diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index bfe06a046..0d9e56d4e 100644
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -3,6 +3,84 @@ name: ParFlow CI Test
 on: [push, pull_request]
 
 jobs:
+  style-check:
+     runs-on: ubuntu-22.04
+     name: Check Style
+     steps:
+       - name: Checkout commit
+         uses: actions/checkout@v4
+
+       - name: Package Install
+         run: |
+            sudo apt-get -qq update
+            sudo apt-get -qq install curl cmake build-essential tcl-dev tk-dev
+
+       - name: Cache dependencies
+         uses: actions/cache@v4
+         id: cache-parflow-style-dependencies
+         env:
+           cache-name: cache-parflow-style-dependencies
+           cache-parflow-style-hit: ${{steps.cache-parflow-style-dependencies.outputs.cache-hit}}
+         with:
+           path: "~/depend"
+           key: cache-check-style-${{ secrets.CACHE_VERSION }}
+
+       - name: Directory Setup
+         run: |
+            mkdir -p $HOME/install
+            echo "PARFLOW_DIR=$HOME/install" >> $GITHUB_ENV
+            echo "Setting PARFLOW_DIR=$HOME/install"
+            mkdir -p $HOME/depend/{include,lib}
+            echo "PARFLOW_DEP_DIR=$HOME/depend" >> $GITHUB_ENV
+            echo "Setting PARFLOW_DEP_DIR=$HOME/depend"
+            echo "PARFLOW_DEP_DIR=$HOME/depend" >> $GITHUB_ENV
+            echo "$HOME/depend/bin" >> $GITHUB_PATH
+            echo "$PARFLOW_DIR/bin" >> $GITHUB_PATH
+            export LD_LIBRARY_PATH=${PARFLOW_DEP_DIR}/lib64:${PARFLOW_DEP_DIR}/lib:${LD_LIBRARY_PATH}
+            echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+
+       - name: Build Uncrustify
+         env:
+           CACHE_HIT: ${{steps.cache-parflow-style-dependencies.outputs.cache-hit}}
+         run: |
+            if [[ "$CACHE_HIT" != 'true' ]]; then
+              echo "cache-check-style-${{ secrets.CACHE_VERSION }}" > ~/depend/cache-key
+              curl -L -o uncrustify-0.79.0.tar.gz https://github.com/uncrustify/uncrustify/archive/uncrustify-0.79.0.tar.gz
+              tar -xf uncrustify-0.79.0.tar.gz
+              cd uncrustify-uncrustify-0.79.0
+              mkdir build
+              cd build
+              cmake -DCMAKE_INSTALL_PREFIX=/usr/local ..
+              make
+              sudo make -j 2 install
+              cd ../..
+              rm -fr uncrustify-uncrustify-0.79.0 uncrustify-0.79.0.tar.gz
+            fi
+
+       - name: CMake Install
+         env:
+           CACHE_HIT: ${{steps.cache-parflow-style-dependencies.outputs.cache-hit}}
+         run: |
+            if [[ "$CACHE_HIT" != 'true' ]]; then
+               echo "Installing"
+               mkdir -p $HOME/depend/{include,lib}
+               cd ~/depend
+               echo "cache-${{ matrix.config.os }}-${{ matrix.config.backend }}-${{ secrets.CACHE_VERSION }}" > ~/depend/cache-key
+               wget -nv --no-check-certificate https://github.com/Kitware/CMake/releases/download/v3.25.1/cmake-3.25.1-linux-x86_64.tar.gz
+               tar -xf cmake-3.25.1-linux-x86_64.tar.gz
+               $HOME/depend/cmake-3.25.1-linux-x86_64/bin/cmake --version
+               echo "$HOME/depend/cmake-3.25.1-linux-x86_64/bin" >> $GITHUB_PATH
+            fi
+
+       - name: ParFlow CMake Configure
+         run: |
+            cmake -Bbuild -DPARFLOW_ENABLE_PYTHON=ON -DCMAKE_INSTALL_PREFIX=$PARFLOW_DIR
+       - name: ParFlow CMake Build
+         run: (cd build; make -j 2 install)
+       - name: Run Uncrustify
+         run: |
+            ./bin/pfformat --check
+
   build:
     name: ${{ matrix.config.name }}
     runs-on: ${{ matrix.config.os }}
@@ -77,7 +155,7 @@ jobs:
           }
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Package Install 22.04
       if: matrix.config.os == 'ubuntu-22.04'
@@ -102,7 +180,7 @@ jobs:
 # settings for the project (using date-timestamp for secret).
 
     - name: Cache dependencies
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: cache-parflow-dependencies
       env:
         cache-name: cache-parflow-dependencies
@@ -194,7 +272,7 @@ jobs:
       if: matrix.config.backend == 'kokkos'
       run: |
         if [[ "$CACHE_HIT" != 'true' ]]; then
-          URL=https://github.com/kokkos/kokkos/archive/refs/tags/3.3.01.tar.gz
+          URL=https://github.com/kokkos/kokkos/archive/refs/tags/4.2.01.tar.gz
           mkdir kokkos
           cd kokkos
           curl -L $URL | tar --strip-components=1 -xz
@@ -302,7 +380,7 @@ jobs:
         gcc -v
         mpicc -v
         if [[ "${{ matrix.config.amps_layer }}" == "oas3" ]]; then HAVE_CLM="OFF"; else HAVE_CLM="ON"; fi
-        CC=mpicc CXX=mpicxx F77=mpif77 FC=mpif90 cmake -Bbuild -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-Wall -Werror -Wno-unused-result -Wno-unused-function" -DPARFLOW_ENABLE_TIMING=TRUE -DPARFLOW_AMPS_LAYER=${{ matrix.config.amps_layer }} -DMPIEXEC_POSTFLAGS='--oversubscribe' -DPARFLOW_ACCELERATOR_BACKEND=${{ matrix.config.backend }} -DPARFLOW_AMPS_SEQUENTIAL_IO=true -DPARFLOW_HAVE_CLM=${HAVE_CLM} -DHYPRE_ROOT=$PARFLOW_DEP_DIR -DOAS3_ROOT=$PARFLOW_DEP_DIR -DSILO_ROOT=$PARFLOW_DEP_DIR -DPARFLOW_ENABLE_PYTHON=${{ matrix.config.python }} -DPARFLOW_PYTHON_VIRTUAL_ENV=${{ matrix.config.python }} $NETCDF_FLAGS $KOKKOS_FLAGS $RMM_FLAGS -DCMAKE_INSTALL_PREFIX=$PARFLOW_DIR
+        CC=mpicc CXX=mpicxx F77=mpif77 FC=mpif90 cmake -Bbuild -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-Wall -Werror -Wno-unused-result -Wno-unused-function" -DPARFLOW_ENABLE_TIMING=TRUE -DPARFLOW_AMPS_LAYER=${{ matrix.config.amps_layer }} -DMPIEXEC_POSTFLAGS='--oversubscribe' -DPARFLOW_ACCELERATOR_BACKEND=${{ matrix.config.backend }} -DPARFLOW_AMPS_SEQUENTIAL_IO=true -DPARFLOW_HAVE_CLM=${HAVE_CLM} -DHYPRE_ROOT=$PARFLOW_DEP_DIR -DOAS3_ROOT=$PARFLOW_DEP_DIR -DSILO_ROOT=$PARFLOW_DEP_DIR -DPARFLOW_ENABLE_PYTHON=${{ matrix.config.python }} $NETCDF_FLAGS $KOKKOS_FLAGS $RMM_FLAGS -DCMAKE_INSTALL_PREFIX=$PARFLOW_DIR
 
     - name: ParFlow CMake Build
       run: (cd build; make -j 2 install)
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
new file mode 100644
index 000000000..98ab1039f
--- /dev/null
+++ b/.github/workflows/macos.yml
@@ -0,0 +1,155 @@
+name: ParFlow MacOS CI Test
+
+#
+# Notes
+# Currently using GCC for the build rather than Clang
+# Python 3.12 used rather than more current available versions since Numba library does not work with later versions
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    name: ${{ matrix.config.name }}
+    runs-on: ${{ matrix.config.os }}
+    strategy:
+      # fail-fast: true
+      matrix:
+        config:
+        - {
+            name: "MacOS 14",
+            os: macos-14,
+            cudaos: '',
+            python: "true",
+            backend: "none",
+            amps_layer: mpi1,
+            netcdf: "true"
+          }
+
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.12' 
+    - uses: fortran-lang/setup-fortran@v1
+      with:
+        compiler: gcc
+    - uses: mpi4py/setup-mpi@v1
+      with:
+        mpi: openmpi
+
+    - name: HDF5 Install
+      run: |
+        brew install hdf5-mpi
+
+# Cache dependencies so don't have to rebuild on each test.
+# Can flush caches by resetting the CACHE_VERSION secret on GitHub
+# settings for the project (using date-timestamp for secret).
+
+    - name: Cache dependencies
+      uses: actions/cache@v4
+      id: cache-parflow-dependencies
+      env:
+        cache-name: cache-parflow-dependencies
+        cache-parflow-hit: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
+      with:
+        path: "~/depend"
+        key: cache-${{ matrix.config.os }}-${{ matrix.config.backend }}-${{ secrets.CACHE_VERSION }}
+
+    - name: Directory Setup
+      run: |
+        mkdir -p $HOME/install
+        echo "PARFLOW_DIR=$HOME/install" >> $GITHUB_ENV
+        echo "Setting PARFLOW_DIR=$HOME/install"
+        mkdir -p $HOME/depend/{include,lib}
+        echo "PARFLOW_DEP_DIR=$HOME/depend" >> $GITHUB_ENV
+        echo "Setting PARFLOW_DEP_DIR=$HOME/depend"
+        echo "PARFLOW_DEP_DIR=$HOME/depend" >> $GITHUB_ENV
+        echo "$HOME/depend/bin" >> $GITHUB_PATH
+        export LD_LIBRARY_PATH=${PARFLOW_DEP_DIR}/lib64:${PARFLOW_DEP_DIR}/lib:${LD_LIBRARY_PATH}
+        echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+        export PARFLOW_PYTHON3=/opt/homebrew/bin/python3.12
+        echo "PARFLOW_PYTHON3=${PARFLOW_PYTHON3}" >> $GITHUB_ENV
+
+    - name: CMake Install
+      if: matrix.config.os != 'macos-14'
+      env:
+        CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
+      run: |
+       if [[ "$CACHE_HIT" != 'true' ]]; then
+          echo "Installing"
+          cd ~/depend
+          echo "cache-${{ matrix.config.os }}-${{ matrix.config.backend }}-${{ secrets.CACHE_VERSION }}" > ~/depend/cache-key
+          wget -nv --no-check-certificate https://github.com/Kitware/CMake/releases/download/v3.25.1/cmake-3.25.1-linux-x86_64.tar.gz
+          tar -xf cmake-3.25.1-linux-x86_64.tar.gz
+          $HOME/depend/cmake-3.25.1-linux-x86_64/bin/cmake --version
+        fi
+        echo "$HOME/depend/cmake-3.25.1-linux-x86_64/bin" >> $GITHUB_PATH
+
+    # Package netcdf is missing features needed by the ParFlow writers, build from source
+    # netcdf is True when we want to build the PF NetCDF writer.
+    - name: NetCDF Install
+      env:
+        CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
+      if: matrix.config.netcdf == 'true'
+      run: |
+        if [[ "$CACHE_HIT" != 'true' ]]; then
+          URL=https://github.com/Unidata/netcdf-c/archive/v4.9.0.tar.gz
+          mkdir netcdf-c
+          cd netcdf-c
+          curl -L $URL | tar --strip-components=1 -xz
+          CC=mpicc CPPFLAGS=-I/opt/homebrew/include LDFLAGS=-L/opt/homebrew/lib/ ./configure --prefix=$PARFLOW_DEP_DIR
+          make
+          make install
+          cd ..
+
+          URL=https://github.com/Unidata/netcdf-fortran/archive/v4.5.4.tar.gz
+          mkdir netcdf-fortran
+          cd netcdf-fortran
+          curl -L $URL | tar --strip-components=1 -xz
+          CC=mpicc FC=mpifort CPPFLAGS=-I${PARFLOW_DEP_DIR}/include LDFLAGS=-L${PARFLOW_DEP_DIR}/lib ./configure --prefix=${PARFLOW_DEP_DIR}
+          make
+          make install
+        fi
+        echo "NETCDF_FLAGS=-DNETCDF_DIR=$PARFLOW_DEP_DIR -DPARFLOW_ENABLE_HDF5=TRUE" >> $GITHUB_ENV
+
+    - name: SILO Install
+      env:
+        CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
+      run: |
+        if [[ "$CACHE_HIT" != 'true' ]]; then
+          URL=https://github.com/LLNL/Silo/archive/refs/tags/4.11.tar.gz
+          mkdir silo
+          cd silo
+          curl -L $URL | tar --strip-components=1 -xz
+          ./configure --disable-silex --disable-hzip --disable-fpzip --prefix=$PARFLOW_DEP_DIR --disable-fortran --with-readline=no
+          make -j 2 install
+        fi
+
+    - name: HYPRE Install
+      env:
+        CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
+      run: |
+        if [[ "$CACHE_HIT" != 'true' ]]; then
+          URL=https://github.com/hypre-space/hypre/archive/v2.26.0.tar.gz
+          mkdir hypre
+          cd hypre
+          curl -L $URL | tar --strip-components=1 -xz
+          cd src
+          CC=mpicc CXX=mpic++ FC=mpif77 F77=mpif90 ./configure --prefix=$PARFLOW_DEP_DIR
+          make -j 2 install
+        fi
+
+    - name: ParFlow CMake Configure
+      run: |
+        mpicc -v
+        if [[ "${{ matrix.config.amps_layer }}" == "oas3" ]]; then HAVE_CLM="OFF"; else HAVE_CLM="ON"; fi
+        CC=mpicc CXX=mpicxx F77=mpif77 FC=mpif90 cmake -Bbuild -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-Wall -Werror -Wno-unused-result -Wno-unused-function" -DPARFLOW_ENABLE_TIMING=TRUE -DPARFLOW_AMPS_LAYER=${{ matrix.config.amps_layer }} -DPARFLOW_ACCELERATOR_BACKEND=${{ matrix.config.backend }} -DPARFLOW_AMPS_SEQUENTIAL_IO=true -DPARFLOW_HAVE_CLM=${HAVE_CLM} -DHYPRE_ROOT=$PARFLOW_DEP_DIR -DOAS3_ROOT=$PARFLOW_DEP_DIR -DSILO_ROOT=$PARFLOW_DEP_DIR -DPARFLOW_ENABLE_PYTHON=${{ matrix.config.python }} $NETCDF_FLAGS $KOKKOS_FLAGS $RMM_FLAGS -DCMAKE_INSTALL_PREFIX=$PARFLOW_DIR -DTCL_INCLUDE_PATH=/opt/homebrew/bin/usr/include -DTCL_LIBRARY=/opt/homebrew/lib/libtcl8.6.dylib -DTCL_TCLSH=/opt/homebrew/bin/tclsh -DPython3_EXECUTABLE=$PARFLOW_PYTHON3
+
+    - name: ParFlow CMake Build
+      run: (cd build; make -j 2 VERBOSE=1 install)
+
+    # Can't test with GPU since have no GPU hardware on testing nodes, GPU test is build only
+    - name: ParFlow CTest
+      if: matrix.config.backend != 'cuda' && matrix.config.backend != 'kokkos'
+      run: |
+        (cd build; ctest --output-on-failure)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index bb45fc071..af913c6eb 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -214,6 +214,7 @@ if ( ${PARFLOW_AMPS_LAYER} IN_LIST PARFLOW_AMPS_LAYER_REQUIRE_MPI )
   if (${MPI_C_FOUND})
     set(PARFLOW_HAVE_MPI "yes")
     set(HAVE_MPI ${PARFLOW_HAVE_MPI})
+    message(STATUS "Found MPIEXEC: ${MPIEXEC}")    
   endif (${MPI_C_FOUND})
 
 endif ( ${PARFLOW_AMPS_LAYER} IN_LIST PARFLOW_AMPS_LAYER_REQUIRE_MPI )
@@ -531,6 +532,44 @@ include_directories ("${PROJECT_SOURCE_DIR}/pfsimulator/amps/common")
 
 include_directories ("${PROJECT_BINARY_DIR}/include")
 
+#-----------------------------------------------------------------------------
+# Building Python key library and documentation if required
+#-----------------------------------------------------------------------------
+
+option(PARFLOW_ENABLE_PYTHON "Build python module for running ParFlow" "FALSE")
+if (PARFLOW_ENABLE_PYTHON)
+  find_package(Python3 3.6 QUIET REQUIRED COMPONENTS Interpreter)
+  set(PARFLOW_PYTHON_DEPENDS)
+  set(PARFLOW_HAVE_PYTHON)
+  set(PARFLOW_PYTHON "${CMAKE_INSTALL_PREFIX}/py-env/bin/python3")
+  message(STATUS "Found Python: ${Python3_EXECUTABLE}")
+  add_custom_command(
+    WORKING_DIRECTORY
+      "${CMAKE_CURRENT_BINARY_DIR}"
+    OUTPUT
+      "${PARFLOW_PYTHON}"
+    COMMAND
+      ${Python3_EXECUTABLE} -m venv ${CMAKE_INSTALL_PREFIX}/py-env
+    COMMAND
+      "${CMAKE_INSTALL_PREFIX}//py-env/bin/pip3" install --upgrade pip wheel setuptools
+    COMMAND
+      "${CMAKE_INSTALL_PREFIX}/py-env/bin/pip3" install -r "${CMAKE_CURRENT_SOURCE_DIR}/pftools/python/requirements_all.txt"
+    COMMAND
+      "${CMAKE_INSTALL_PREFIX}//py-env/bin/pip3" install -r "${CMAKE_CURRENT_SOURCE_DIR}/docs/user_manual/requirements.txt"
+    COMMAND
+      "${CMAKE_INSTALL_PREFIX}//py-env/bin/pip3" install -r "${CMAKE_CURRENT_SOURCE_DIR}/pftools/python/requirements_dev.txt"
+    COMMENT
+      "Generate ParFlow virtual environment"
+    VERBATIM
+    )
+    add_custom_target(
+      BuildVirtualEnv
+      DEPENDS
+        "${PARFLOW_PYTHON}"
+    )
+    set(PARFLOW_PYTHON_DEPENDS BuildVirtualEnv)
+endif ()
+
 #-----------------------------------------------------------------------------
 # Setup configure.h file for accessing configure options
 # -----------------------------------------------------------------------------
@@ -609,46 +648,10 @@ if ( ${PARFLOW_ENABLE_DOXYGEN} )
 endif()
 
 #-----------------------------------------------------------------------------
-# Building Python key library and documentation if required
+# Python directories
 #-----------------------------------------------------------------------------
 
-option(PARFLOW_ENABLE_PYTHON "Build python module for running ParFlow" "FALSE")
 if (PARFLOW_ENABLE_PYTHON)
-  find_package(Python3 3.6 QUIET REQUIRED COMPONENTS Interpreter)
-  set(PARFLOW_PYTHON ${Python3_EXECUTABLE})
-  set(PARFLOW_PYTHON_DEPENDS)
-  set(PARFLOW_HAVE_PYTHON)
-
-  option(PARFLOW_PYTHON_VIRTUAL_ENV "Use local Python virtual environment" "FALSE")
-  if (PARFLOW_PYTHON_VIRTUAL_ENV)
-    set(PARFLOW_PYTHON "${CMAKE_CURRENT_BINARY_DIR}/py-env/bin/python3")
-    add_custom_command(
-      WORKING_DIRECTORY
-        "${CMAKE_CURRENT_BINARY_DIR}"
-      OUTPUT
-        "${PARFLOW_PYTHON}"
-      COMMAND
-        ${Python3_EXECUTABLE} -m venv py-env
-      COMMAND
-        "${CMAKE_CURRENT_BINARY_DIR}/py-env/bin/pip3" install --upgrade pip wheel setuptools
-      COMMAND
-        "${CMAKE_CURRENT_BINARY_DIR}/py-env/bin/pip3" install -r "${CMAKE_CURRENT_SOURCE_DIR}/pftools/python/requirements_all.txt"
-      COMMAND
-        "${CMAKE_CURRENT_BINARY_DIR}/py-env/bin/pip3" install -r "${CMAKE_CURRENT_SOURCE_DIR}/docs/user_manual/requirements.txt"
-      COMMAND
-        "${CMAKE_CURRENT_BINARY_DIR}/py-env/bin/pip3" install -r "${CMAKE_CURRENT_SOURCE_DIR}/pftools/python/requirements_dev.txt"
-      COMMENT
-        "Generate virtual environment for building Python components"
-      VERBATIM
-    )
-    add_custom_target(
-      BuildVirtualEnv
-      DEPENDS
-        "${PARFLOW_PYTHON}"
-    )
-    set(PARFLOW_PYTHON_DEPENDS BuildVirtualEnv)
-  endif()
-
   add_subdirectory(pf-keys)
   add_subdirectory(pftools/python)
   add_subdirectory(test/python)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 9c4ab1490..d86fe138a 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,26 +1,30 @@
 # Contributing to ParFlow
 
-**This is a DRAFT but should hopefully provide some helpful pointers**
-
 This file is inspired by Atom's CONTRIBUTING.md file.
 
-:+1::tada: First off, thanks for taking the time to contribute! :tada::+1:
+Thanks for taking the time to contribute!
 
 The following is a set of guidelines for contributing to ParFlow.
 ParFlow is hosted on [ParFLow Github](https://github.com/parflow)
 site. These are mostly guidelines, not rules. Use your best judgment,
 and feel free to propose changes to this document in a pull request.
 
+If you are new to contributing to open source projects there are many
+general guides available to understand the basic ideas and processes.
+A large collection is available
+[here](https://github.com/freeCodeCamp/how-to-contribute-to-open-source).
+
 #### Table Of Contents
 
 [I just have a question, where can I ask it?](#i-have-a-question)
 
+[What should I know before I get started?](#what-should-i-know-before-i-get-started)
+
 [How Can I Contribute?](#how-can-i-contribute)
   * [Reporting Bugs](#reporting-bugs)
   * [Suggesting Enhancements](#suggesting-enhancements)
   * [Your First Code Contribution](#your-first-code-contribution)
-
-[What should I know before I get started?](#what-should-i-know-before-i-get-started)
+  * [Pull Requests](#pull-requests)
 
 ## I have a question
 
@@ -28,26 +32,26 @@ and feel free to propose changes to this document in a pull request.
 faster results by using the resources below.  Issues are meant for bug
 reports and feature requests.
 
-The [Parflow-Users](https://groups.google.com/g/parflow) is
+The [ParFlow-Users](https://groups.google.com/g/parflow) is
 seen by many ParFlow users and developers.  You can also subscribe to
 the email list.  The traffic is minimal and is the platform for
 discussing ParFlow questions.
 
 The ParFlow GitHub wiki is where the team is starting to assemble a list of frequently asked questions:
-- [Parflow FAQ](https://github.com/parflow/parflow/wiki/ParFlow-FAQ)
+- [ParFlow FAQ](https://github.com/parflow/parflow/wiki/ParFlow-FAQ)
 
 A ParFlow blog is available with notes from users on how to compile and use ParFlow:
-- [Parflow Blog](http://parflow.blogspot.com/)
+- [ParFlow Blog](http://parflow.blogspot.com/)
 
-To report Parflow bugs or make change requests, please use the GitHub issue tracker for Parflow:
-- [Parflow Issue Tracker](https://github.com/parflow/parflow/issues)
+To report ParFlow bugs or make change requests, please use the GitHub issue tracker for ParFlow:
+- [ParFlow Issue Tracker](https://github.com/parflow/parflow/issues)
 
 ## How Can I Contribute?
 
 ### Reporting Bugs
 
 This section guides you through submitting a bug report for
-Parflow. Following these guidelines helps maintainers and the community
+ParFlow. Following these guidelines helps maintainers and the community
 understand your report, reproduce the behavior, and find related reports.
 
 Before creating bug reports, please check [this
@@ -74,21 +78,21 @@ additional details to help maintainers reproduce the problem:
 
 * **Use a clear and descriptive title** for the issue to identify the problem.
 * **Describe the exact steps which reproduce the problem** in as many details as possible. 
-* **Providing specific examples of Parflow input that demonstrate the problem helps enormously in debugging**. Include links to files or GitHub projects, or copy/paste-able snippets, which you use in those examples.  Developers unable to recreate a problem makes it very hard to fix.
+* **Providing specific examples of ParFlow input that demonstrate the problem helps enormously in debugging**. Include links to files or GitHub projects, or copy/paste-able snippets, which you use in those examples.  Developers unable to recreate a problem makes it very hard to fix.
 * **Describe the behavior you observed after following the steps** and point out what exactly is the problem with that behavior.
 * **Explain which behavior you expected to see instead and why.**
-* **If you're reporting that Parflow crashed**, include a stack trace if available. 
-* **If you're reporting that CMake doesn't configure or Parflow doesn't compile**, include the output from the CMake configure and the compiler output.  "It doesn't compile" is not enough information for the team to help determine what has gone wrong.
+* **If you're reporting that ParFlow crashed**, include a stack trace if available. 
+* **If you're reporting that CMake doesn't configure or ParFlow doesn't compile**, include the output from the CMake configure and the compiler output.  "It doesn't compile" is not enough information for the team to help determine what has gone wrong.
 * **Large files** can be attached to the issue via a [file attachment](https://help.github.com/articles/file-attachments-on-issues-and-pull-requests/), or put it in a [gist](https://gist.github.com/) and provide link to that gist.
 
 Provide more context by answering these questions:
 
-* **Did the problem start happening recently** (e.g. after updating to a new version of Parflow) or was this always a problem?
+* **Did the problem start happening recently** (e.g. after updating to a new version of ParFlow) or was this always a problem?
 * **Can you reliably reproduce the issue?** If not, provide details about how often the problem happens and under which conditions it normally happens.
 
 Include details about your configuration and environment:
 
-* **Which version of Parflow are you using?** You can get the exact version by running `parflow -v`.
+* **Which version of ParFlow are you using?** You can get the exact version by running `parflow -v`.
 * **What OS and version are you using**?
 * **What compiler and version are you using**?
 * **What OS and version are you using**?
@@ -96,7 +100,7 @@ Include details about your configuration and environment:
 ### Suggesting Enhancements
 
 This section guides you through submitting an enhancement suggestion
-for Parflow, including completely new features and minor improvements to
+for ParFlow, including completely new features and minor improvements to
 existing functionality. Following these guidelines helps maintainers
 and the community understand your suggestion and find related
 suggestions.
@@ -119,58 +123,64 @@ Enhancement suggestions are tracked as [GitHub issues](https://guides.github.com
 * **Provide a step-by-step description of the suggested enhancement** in as many details as possible.
 * **Provide specific examples to demonstrate the steps**. Include copy/paste-able snippets which you use in those examples, as [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines).
 * **Describe the current behavior** and **explain which behavior you expected to see instead** and why.
-* **Explain why this enhancement would be useful** to most Parflow users.
+* **Explain why this enhancement would be useful** to most ParFlow users.
 
 ## What should I know before I get started in the code?
 
 ### Code organization
 
-The Parflow is split into two main parts.  The pftools directory
-contains the input parsing (written using TCL) and the pfsimulator
-directory which contains the simulator code.  Parflow is written as a
-library so it can be embedded inside other applications; the code for
-the library is in in the pfsimulator/parflow_lib directory.  For
-stand-along execution an executable ('main') is also supplied; the
-code for the stand-alone application is in pfsimulator/parflow_exe.
-Parflow can optionally be linked with CLM; the code for for CLM is in
-pfsimulator/clm.  Parflow has an embedded a copy of KINSOL which is in
-the pfsimulator/kinsol directory.
-
-Parflow was written before the MPI standard was created.  In order to
+The ParFlow is split into two main parts.  The pftools directory
+contains the input parsing (written using Python or TCL) and the
+pfsimulator directory which contains the simulator code.  The
+stand-along executable code ('main') is in pfsimulator/parflow\_exe.
+Most of ParFlow is written as a library so it can be embedded inside other
+applications; the code for the library is in in the
+pfsimulator/parflow_lib directory. ParFlow can optionally be linked
+with CLM; the code for for CLM is in pfsimulator/clm.  ParFlow has an
+embedded a copy of KINSOL which is in the pfsimulator/kinsol
+directory.
+
+ParFlow was written before the MPI standard was created.  In order to
 easily port to vendor specific message passing libraries a
 communication layer was created called Another Message Passing Sytem
 (AMPS).  We continue to use AMPS since it has proved useful for easily
-porting Parflow to use with other frameworks, like [OASIS 3](
+porting ParFlow to use with other frameworks, like [OASIS 3](
 https://journals.ametsoc.org/doi/10.1175/MWR-D-14-00029.1).  The AMPS
 library is in pfsimulator/amps.
 
-The pftools directory contains the TCL interface and a verity of tools
-for pre and post processing.  
+The pftools directory contains the Python and TCL interfaces and a
+verity of tools for pre/post processing input/output files.  The
+normal workflow for ParFlow is to use the pftools interface (Python or
+TCL) to create an input for the ParFlow executable (these files use
+the '.pfidb' file extenstion by convention).
 
-Additional documentation can be found in the [Parflow Users
+Additional documentation can be found in the [ParFlow Users
 Manaul](https://github.com/parflow/parflow/blob/master/parflow-manual.pdf)
 
-### Parflow Input and Input Database
-
-Parflow was written to be principally driven by an input file written in
-TCL.  This was done to enable a much more feature rich input file than
-a standard XML/JSON file.  Expressions can be used for computing
-parameters and for loops can be used for easier setup up large numbers
-of similar input elements.  The TCL was also meant to capture the
-problem workflow in an executable script for repeat-ability.  For
-example, data post-processing could be part of a standard input script.
-Jupyter Notebooks or similar systems are examples of the intent;
-Parflow was written before the current workflow technologies existed
-so we used a scripting language.  TCL was chosen since it was widely
-used at the time.
-
-A Parflow input file is simply a set of key/value assignments which go
-into a 'Parflow Input Database'.  The pfset commands in the TCL file
-set the key/value pairs.  This database is saved to a simple text file
-with the key/values when the pfrun command is executed in the TCL
-script.  The '<runame>.pfidb' file is read by the main Parflow
-executable to setup a problem.  The 'pfidb' file is similar in intent
-to a JSON file (we would likely use JSON if starting today).
+### ParFlow Input and Input Database
+
+ParFlow was written to be principally driven by an input file written
+in Python or TCL.  This was done to enable a much more feature rich
+input file than a standard XML/JSON file.  Expressions can be used for
+computing parameters and for loops can be used for easier setup up
+large numbers of similar input elements.  The Python/TCL scripts were
+also meant to capture the problem workflow in an documented script for
+repeatability.  For example, data post-processing could be part of a
+standard input script.  Jupyter Notebooks or similar systems are
+examples of the intent; ParFlow was written before the current
+workflow technologies existed so we used a scripting language.  TCL
+was initially chosen since it was widely used at the time and a Python
+interface has been added.
+
+A ParFlow input file is simply a set of key/value assignments which go
+into a 'ParFlow Input Database'.  The pfset commands in the TCL file
+set the key/value pairs.  The Python input is a bit more structured,
+see the users manual and test cases for examples.  This database is
+saved to a simple text file with the key/values when the pfrun command
+is executed in the TCL script.  The '<runame>.pfidb' file is read by
+the main ParFlow executable to setup a problem.  The 'pfidb' file is
+similar in intent to a JSON file (we would likely use JSON if starting
+today).
 
 This input dataset file (pfidb file) is used by the main parflow
 executable to configure and setup a problem.  The keys are read by
@@ -178,16 +188,18 @@ each ParFlow module to setup a problem.  The key names are a
 structured hierarchy separated by '.'.  When adding new modules, new
 keys can be added to the input database to configure the new module.
 
-### Parflow Modules
-
-Parflow modules are a way to provide a bit of object oriented
-structure in C.  A Parflow module is simply a structures with
-function pointers and data.  The module has a public and instance
-structure.  The basic use for public is functions/data that are
-instance independent.   Instance data is used for functions/data 
-that are specific to an instance.  Frequently Parflow only has a
-single instance of a module so this distinction does not matter.  It
-is still a good idea to split things out to future proof the
+### ParFlow Modules
+
+ParFlow modules are a way to provide a bit of object oriented
+structure in C.  ParFlow was writtenb before C++ was widely available
+on HPC systems.  A ParFlow module is simply a structure with function
+pointers and data.  The module has a public and instance structure.
+The basic use for public is functions/data that are instance
+independent (similiar to class methods/members in C++).  Instance data
+is used for functions/data that are specific to an instance.  In
+theory one can have many instances of a specific type, this is used in
+a few places in ParFlow but most modules only have a single instance.
+It is still a good idea to split things out to future proof the
 implementation in case multiple instance are desired at some point.
 
 The module pattern provides methods for constructing and destructing
@@ -196,8 +208,50 @@ the class of module that they are (e.g. linear solver).
 
 For anyone with an object oriented background this should all sound
 pretty familiar.  Since C doesn't have the language constructs it
-makes the infrastructure for supporting OO more visible (and error
-prone).
+makes the infrastructure for supporting OO more visible, manual and
+error prone.
+
+## Automatic style checking
+
+ParFlow uses automated code formatters to check code as one of the
+checks in our CI system.  Incorrectly formatted code will fail the
+check and won't be merged.  The check is done using Uncrustify for C
+and Black for Python.  A script is provided to automatically reformat 
+code and do the check.
+
+To check if the coding style is correct run the pfformat script at the 
+root of the PF source tree:
+
+```
+cd parflow
+./bin/pfformat --check
+```
+
+Checking is done with the same script:
+
+```
+cd parflow
+./bin/pfformat
+```
+
+You must have installed Uncrustify version 0.79.0 in order for this to
+work.  The Python Black code formatter will be installed during the
+ParFlow installation so you don't need to manually install Black.
+
+Information and source on these tools can be found here:
+
+[Uncrustify](https://github.com/uncrustify/uncrustify)
+
+[Black](https://github.com/psf/black)
+
+### Why auto code formatting?
+
+Enforcing a common code indentation and other style format is fairly
+common on Open Source projects.  Everyone has different preferred
+styles and has setup their editors with different options.  Reviewing
+and merging code conflicts that have divered primarily due to code
+formatting creates unnecessary headaches and makes it harder to
+understand what 'real' changes have occured.
 
 ## ParFlow C Conventions
 
@@ -207,23 +261,14 @@ be usable with C++.  CLM is written in FORTRAN.
 
 ### Indentation and whitespace
 
-Parflow doesn't have a full coding style guide document; follow the
+ParFlow doesn't have a full coding style guide document; follow the
 existing coding style to when doing a code submission.
 
-Code contributions should NOT do indentation/whitespace changes
-intermixed with bug fixes/feature additions.  Large amounts of white
-spaces changes makes it hard for reviewers to see what is actually
-being changed.  Looking at 20 changed files for a 1 line of a bug fix
-is not reviewer friendly.
-
-If you have indentation/whitespace changes, please submit those as a
-separate pull request so they can be easily reviewed.
-
 ParFlow uses Uncrustify to perform automatic code indentation/spacing.
 The bin/pfuncrustify script is supplied to run the uncrustify tool on
-the Parflow source code.  The Uncrustify setup is in the
+the ParFlow source code.  The Uncrustify setup is in the
 bin/parflow.cfg file.  Code contributions should use the the
-pfuncrustify tool to maintain consistency.
+pfformat tool to maintain consistency.
 
 #### Source Code Documentation
 
@@ -293,16 +338,14 @@ Python.
 base for naming conventions, whitespace and other basic coding style
 issues.
 
-[Google
-style](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstringsa)
+[Googlestyle](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstringsa)
 Python docstrings formatting should be used.  Please document every
 method, ParFlow has not done this historically and it has made the
 code more diffult to maintain.  We are documenting now so don't follow
 the bad practices from our past.
 
 Using the [Black](https://pypi.org/project/black/) code formatter is
-highly recommended and we are planing on enforcing use of Black at
-some point.
+enforced in the CI process for checking pull requests.
 
 ## Your First Code Contribution
 
@@ -365,6 +408,8 @@ We will not accept code without a proper license statement.
 
 #### Testing
 
+> **Note:** ParFlow will not accept Pull Requests that do not pass ALL tests
+
 Contributions with feature addtions should include tests for that
 feature.  Submissions that do not pass the test suite will not be
 accepted.  A major goal is to ensure the master branch of ParFlow
@@ -381,17 +426,26 @@ The test suite will run using CTest and results will show how many
 tests pass and fail.
 
 Automated ParFlow testing is done with every pull request using the
-TravisCI continuous integration system.  The results will appear on
-the pull request page on GitHub.  You can view the TravisCI results by
-selecting "Show all checks".  The test system runs serial as well as
-parallel problems.  The current ParFlow regression test suite is
-limited but includes tests on saturated and unsaturated subsurface
-flow and coupled ParFlow CLM systems.  The TravisCI test is run in a
-Linux image based on Ubuntu.  The testing setup is included in the
-.travis.yaml file located in the root ParFlow directory.
-
-
-
-
-
-
+Github Action system.  The results will appear on the pull request
+page on GitHub.  You can view the CI results by selecting "Show all
+checks".  The test system runs serial as well as parallel problems.
+The current ParFlow regression test suite is limited but includes
+tests on saturated and unsaturated subsurface flow and coupled ParFlow
+CLM systems.  The GitHub Action test is run in a Linux image based on
+Ubuntu.  The testing setup can be found in the
+[linux.yml](/.github/workflows/linux.yml) file.
+
+### Pull Requests
+
+ParFlow uses a standard GitHub Fork-Branch-Pull workflow.  If you are
+new to GitHub workflows the [quickstart
+guide](https://docs.github.com/en/get-started/quickstart/contributing-to-projects)
+will walk you through the process.
+
+ParFlow uses this workflow to support several goals:
+
+- Engage the community
+- Ensure ParFlow does not break
+- Maintain ParFlow's quality
+- Fix problems that are important to users
+- Provide a sustainable mechanism for accepting code contributions
diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
index f9eef1c02..67f2ef682 100644
--- a/CONTRIBUTORS.md
+++ b/CONTRIBUTORS.md
@@ -79,3 +79,15 @@ Kitware, Inc
 **Jaro Hokkanen**
 IBG-3, Forschungszentrum Juelich,
 Juelich, Germany
+
+**George Artavanis**
+Princeton Unversity
+ga6@princeton.edu
+
+**Chen Yang**
+Princeton University
+chen_yang@princeton.edu
+
+**Benjamin West**
+University of Arizona
+benjaminwest@arizona.edu
diff --git a/Dockerfile b/Dockerfile
index 1f8613d0c..ca34a3fdf 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -137,7 +137,6 @@ RUN mkdir -p build && \
        -DPARFLOW_ENABLE_TIMING=TRUE \
        -DPARFLOW_HAVE_CLM=TRUE \
        -DPARFLOW_ENABLE_PYTHON=TRUE \
-       -DPARFLOW_PYTHON_VIRTUAL_ENV=ON \
        -DCURL_LIBRARY=/usr/lib/x86_64-linux-gnu/libcurl.so.4 \
        -DCMAKE_INSTALL_PREFIX=$PARFLOW_DIR && \
      make install && \
diff --git a/README-Spack.md b/README-Spack.md
index a786eb4b8..52f0a25bd 100644
--- a/README-Spack.md
+++ b/README-Spack.md
@@ -43,13 +43,13 @@ compile the GNU compiler suite under Spack and avoid using Clang.
 Building GCC will take considerable amount of time.
 
 ```shell
-spack install gcc@10.3.0
+spack install gcc@12.2.0
 ```
 
 Add the compiler to the set of compilers Spack can use:
 
 ```shell
-spack load gcc@10.3.0
+spack load gcc@12.2.0
 spack compiler find
 spack compilers
 ```
diff --git a/README.md b/README.md
index 3f6c835ec..be84d0fe2 100644
--- a/README.md
+++ b/README.md
@@ -257,7 +257,7 @@ An HTML version of the user manual for Parflow may be built using:
 
 ```shell
 cd docs/user_manual
-pip install -r requirements.txt
+pfpython -m pip install -r requirements.txt
 
 make html
 ```
@@ -281,7 +281,7 @@ An HTML version of the user manual for Parflow may be built using:
 
 ```shell
 cd docs/user_manual
-pip install -r requirements.txt
+pfpython -m pip install -r requirements.txt
 
 make latexpdf
 ```
@@ -315,7 +315,6 @@ HTML pages are generated in the build/docs/doxygen/html directory.
       -D PARFLOW_ENABLE_SIMULATOR=OFF \
       -D PARFLOW_ENABLE_KEYS_DOC=ON \
       -D PARFLOW_ENABLE_PYTHON=ON \
-      -D PARFLOW_PYTHON_VIRTUAL_ENV=ON
 
     cd ./build-docker && make ParFlowKeyDoc
 ```
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index e47c4d1f8..6c908b397 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -1,3 +1,92 @@
+# ParFlow Release Notes 3.13.0
+------------------------------
+
+This release contains several bug fixes and minor feature updates.
+
+ParFlow development and bug-fixes would not be possible without contributions of the ParFlow community.  Thank you for all the great contributions.
+
+## Overview of Changes
+
+## User Visible Changes
+
+### Building ParFlow-CLM only is supported
+
+Configuration flags have been added to support building only ParFlow-CLM for use cases where only CLM is desired.
+
+### Kokkos version support updated to version 4.2.01
+
+The Kokkos supported version has been updated to version 4.2.01.  This is the version used in our regression suite.  Other versions may or may not work.
+
+### OASIS version support updated to version 5.1.
+
+The OASIS version used in the regression test suite was updated to OASIS 5.1.
+
+### vegm file reading performance improvements in Python pftools
+
+Improve speed of reading large vegm files in the Python read_vegm function.
+
+### Documentation updates
+
+Clarified top flux boundary conditions (e.g., OverlandFlow, SeepageFace) and EvapTrans files sign convention.  Typo in Haverkamp saturation formula: alpha replaced with A.  Key names "TensorByFileX" renamed to the correct "TensorFileX" key name.
+
+## Bug Fixes
+
+### Python pftools StartCount incorrect bounds check
+
+The StartCount input value was incorrectly checked to be -1 or 0.  This prevented setting to larger value for doing a restart.  Larger values are now allowed.
+
+### Python pftools reading porosity from a file not working
+
+The Python input was throwing an error if the type for the porosity input was set to PFBFile.   This has been fixed and using a PFB file for input should work in Python.
+
+### Memory corruption when using the PressureFile option
+
+The PressureFile option (and others) were causing memory corruption due to multiple frees of the filenames. Removed the incorrect free calls in the shutdown logic.  This will fix some segmentation issues seen by users.
+
+### 
+
+## Internal/Developer Changes
+
+### Direchelt boundary condition fix in nl_function_eval
+
+Z_mult was incorrectly being divided by 2 in nl_function_eval.
+
+### GitHub Actions updated
+
+The CI testing suite was using out-dated GitHub Action modules; the modules have been updated.
+
+### Added Python CI test result checks
+
+The Python tests were incorrectly not checking results of runs and passing if the test ran.   Checks have been added as in the TCL test suite to check output results for regressions.
+
+See the `pf_test_file` and `pf_test_file_with_abs` Python methods.
+
+### Python CI tests for optional external package dependencies
+
+Python CI tests are now guarded for optional package dependencies such as Hypre, Silo, etc.
+
+See the `pf_test_file` and `pf_test_file_with_abs` Python methods.
+
+### Compilation with Intel-OneAPI compiler fixes
+
+The Intel-OneAPI compiler with fast-floating-point mode does not support isnan() (always evaluates to false).   NaN sentinel value was replaced with FLT_MIN.
+
+### Improvements to C/C++ standards compliance
+
+Minor code cleanup to remove old K&R style definitions and declarations.
+
+### Updated etrace
+
+Update the etrace script to work with Python3.
+
+### 
+
+## Known Issues
+
+See https://github.com/parflow/parflow/issues for current bug/issue reports.
+
+
+********************************************************************************
 # ParFlow Release Notes 3.12.0
 ------------------------------
 
diff --git a/VERSION b/VERSION
index b0ab92d42..017d61fcd 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-v3.12.0
+v3.13.0
diff --git a/bin/parflow.cfg b/bin/parflow.cfg
index 990a29cfd..f1a103efb 100644
--- a/bin/parflow.cfg
+++ b/bin/parflow.cfg
@@ -20,16 +20,32 @@ indent_switch_case              = 2
 align_func_params               = true
 
 cmt_indent_multi                = true
-cmt_star_cont         		= TRUE
+cmt_star_cont         		= true
 
 # This will align trailing \ on macro defines
-align_nl_cont                   = TRUE    # true/false
+align_nl_cont                   = 3
 
 # Use Unix LF line endings
 newlines                        = LF
 
 # Convert all tabs to spaces in comments.
-cmt_convert_tab_to_spaces       = True
+cmt_convert_tab_to_spaces       = true
+
+# Indent function call params to the '('
+indent_func_call_param          = false
+
+indent_var_def_cont=false
+indent_func_def_param=false
+indent_func_proto_param=false
+indent_func_class_param=false
+indent_func_ctor_var_param=false
+indent_template_param=false
+indent_paren_nl=true
+
+# ParFlow Additions
+
+# The span for aligning on '#define' bodies (0=don't align)
+align_pp_define_span            = 0      # number
 
 # End ParFlow deviations from GNU and custom uncrustify
 
@@ -51,20 +67,21 @@ nl_for_brace			= add		# "for () {" vs "for () \n {"
 nl_else_brace			= add		# "else {" vs "else \n {"
 nl_while_brace			= add		# "while () {" vs "while () \n {"
 nl_switch_brace			= add		# "switch () {" vs "switch () \n {"
-nl_func_var_def_blk		= 1
+# nl_func_var_def_blk		= 1
+nl_var_def_blk_end_func_top     = 1
 nl_before_case			= 1
 nl_fcall_brace			= add		# "foo() {" vs "foo()\n{"
 nl_fdef_brace			= add		# "int foo() {" vs "int foo()\n{"
-# nl_after_return			= TRUE
+# nl_after_return			= true
 nl_brace_while			= add
 nl_brace_else			= add
-nl_squeeze_ifdef		= TRUE
+nl_squeeze_ifdef		= true
 
 # mod_paren_on_return		= ignore	# "return 1;" vs "return (1);"
-# mod_full_brace_if		= ignore	# "if (a) a--;" vs "if (a) { a--; }"
-# mod_full_brace_for		= ignore	# "for () a--;" vs "for () { a--; }"
-# mod_full_brace_do		= ignore	# "do a--; while ();" vs "do { a--; } while ();"
-# mod_full_brace_while		= ignore	# "while (a) a--;" vs "while (a) { a--; }"
+#mod_full_brace_if		= force		# "if (a) a--;" vs "if (a) { a--; }"
+#mod_full_brace_for		= force		# "for () a--;" vs "for () { a--; }"
+#mod_full_brace_do		= force		# "do a--; while ();" vs "do { a--; } while ();"
+#mod_full_brace_while		= force		# "while (a) a--;" vs "while (a) { a--; }"
 
 sp_before_semi			= remove
 sp_paren_paren			= remove	# space between (( and ))
@@ -89,39 +106,30 @@ sp_func_def_paren		= remove	# "int foo (){" vs "int foo(){"
 sp_func_call_paren		= remove	# "foo (" vs "foo("
 sp_func_proto_paren		= remove	# "int foo ();" vs "int foo();"
 
-indent_func_call_param          = false
-
-indent_var_def_cont=true
-indent_func_call_param=false
-indent_func_def_param=false
-indent_func_proto_param=false
-indent_func_class_param=false
-indent_func_ctor_var_param=false
-indent_template_param=false
-indent_paren_nl=true
-
 # align_with_tabs			= FALSE		# use tabs to align
 # align_on_tabstop		= FALSE		# align on tabstops
 # align_enum_equ_span		= 4
-# align_nl_cont			= TRUE
+# align_nl_cont			= true
 # align_var_def_span		= 2
-# align_var_def_inline		= TRUE
-# align_var_def_star		= TRUE
-# align_var_def_colon		= TRUE
+# align_var_def_inline		= true
+# align_var_def_star		= true
+# align_var_def_colon		= true
 # align_assign_span		= 1
 # align_struct_init_span		= 3
 # align_var_struct_span		= 3
 # align_right_cmt_span		= 3
 # align_pp_define_span		= 3
 # align_pp_define_gap		= 4
-# align_number_left		= TRUE
+# align_number_left		= true
 # align_typedef_span		= 5
 # align_typedef_gap		= 3
 
-# cmt_star_cont			= TRUE
+# cmt_star_cont			= true
+
+eat_blanks_before_close_brace	= true
+eat_blanks_after_open_brace	= true
 
-eat_blanks_before_close_brace	= TRUE
-eat_blanks_after_open_brace	= TRUE
 
+# indent_continue                 = indent_columns
 
 
diff --git a/bin/pfformat b/bin/pfformat
index 29452a9ed..ea133fca2 100755
--- a/bin/pfformat
+++ b/bin/pfformat
@@ -15,7 +15,7 @@
 
 OPTIND=1         # Reset in case getopts has been used previously in the shell.
 
-UNCRUSTIFY_VERSION='0.67'
+UNCRUSTIFY_VERSION='0.79.0'
 
 # Exit script
 # $1 (required): Message to print on exit
@@ -64,6 +64,9 @@ function parse_params() {
             -v|--verbose)
                 verbose=true
                 ;;
+            -c|--check)
+                check=true
+                ;;
             *)
                 script_exit "Invalid parameter was provided: $param" 2
                 ;;
@@ -82,19 +85,63 @@ function check_version() {
    esac
 }
 
+function log () {
+    if $verbose ; then
+        printf "$@"
+    fi
+}
+
 function main() {
 
-   parse_params "$@"
+   verbose=false
+   check=false
 
+   parse_params "$@"
+   
    check_version
 
-   files=$(find . -name \*.[ch] | grep -v './build\|third-party')
-   for i in $files
+   failed=false
+
+   for i in $(find . -name \*.[ch] | grep -v './build\|third_party\|./py-env')
    do
-      uncrustify -l C --replace --no-backup -c bin/parflow.cfg $i 
+      if $check
+      then
+	 if output=$(uncrustify --check -c bin/parflow.cfg $i)
+	 then
+	    log 'pfformat PASSED %s\n' $i
+	 else
+	    printf 'pfFormat FAILED\nUncrustify output was «%s»\n' "$output\n"
+	    failed=true
+	 fi
+      else
+	 uncrustify --if-changed --replace --no-backup -c bin/parflow.cfg $i
+      fi
    done
+
+   # Disable this for now while Python is being refactored.
+   if $check
+   then
+      if output=$($PARFLOW_DIR/bin/pfpython -m black . --check)
+      then
+	 log 'pfformat PASSED %s\n' $i
+      else
+	 printf 'pfFormat FAILED\nPython black output was «%s»\n' "$output\n"
+	 failed=true
+      fi
+   else
+      $PARFLOW_DIR/bin/pfpython -m black .
+   fi
+
+   if $check
+   then
+      if $failed ;
+      then
+	 printf 'pfFormat style check : FAILED\n'
+	 exit -1
+      else
+	 printf 'pfFormat style check : PASSED\n'
+      fi
+   fi
 }
 
 main "$@"
-
-
diff --git a/bin/valgrind.sup b/bin/valgrind.sup
index 98d9b45fd..041bbe037 100644
--- a/bin/valgrind.sup
+++ b/bin/valgrind.sup
@@ -17,6 +17,7 @@
 # $HEADER$
 #
 
+
 ###############################################################
 #
 # OPAL suppressions
@@ -115,7 +116,6 @@
   fun:utcp_msg_wait
 }
 
-
 #&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
 #&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
 
@@ -177,6 +177,28 @@
    fun:PMPI_Init	
 }
 
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Param
+   writev(vector[1])
+   ...
+   fun:clone
+}
+
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:malloc
+   ...
+   fun:patcher_open
+   fun:open_components
+   fun:mca_base_framework_components_open
+   fun:opal_memory_base_open
+   fun:mca_base_framework_open
+   ...
+}
+
 {
    Clone
    Memcheck:Leak
@@ -247,6 +269,97 @@
    ...
 }
 
+
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Cond
+   fun:opal_interval_tree_reader_get_token
+   fun:opal_interval_tree_traverse
+   fun:mca_rcache_base_vma_tree_find_all
+   fun:mca_rcache_base_vma_find_all
+   fun:mca_rcache_rgpusm_finalize
+   fun:mca_rcache_base_module_destroy
+   fun:mca_btl_smcuda_del_procs
+   fun:mca_bml_r2_del_procs
+   fun:mca_pml_ob1_del_procs
+   fun:ompi_mpi_finalize
+   ...
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Cond
+   fun:mca_btl_smcuda_sendi
+   fun:mca_bml_base_sendi
+   fun:mca_pml_ob1_send_inline
+   fun:mca_pml_ob1_isend
+   fun:PMPI_Isend
+   ...
+}
+
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Cond
+   fun:mca_btl_smcuda_sendi
+   fun:mca_bml_base_sendi
+   fun:mca_pml_ob1_send_inline
+   fun:mca_pml_ob1_send
+   fun:PMPI_Send
+   ...
+}
+
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Cond
+   fun:mca_btl_smcuda_sendi
+   fun:mca_bml_base_sendi
+   fun:mca_pml_ob1_send_inline
+   fun:mca_pml_ob1_send
+   fun:PMPI_Send
+   fun:hypre_DataExchangeList
+   fun:hypre_StructAssumedPartitionCreate
+   fun:hypre_BoxManAssemble
+   fun:hypre_StructGridAssemble
+   fun:HypreAssembleGrid
+   fun:PFMGInitInstanceXtra
+   fun:KinsolPCInitInstanceXtra
+}
+
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Value8
+   fun:opal_interval_tree_reader_return_token
+   fun:opal_interval_tree_traverse
+   fun:mca_rcache_base_vma_tree_find_all
+   fun:mca_rcache_base_vma_find_all
+   fun:mca_rcache_rgpusm_finalize
+   fun:mca_rcache_base_module_destroy
+   fun:mca_btl_smcuda_del_procs
+   fun:mca_bml_r2_del_procs
+   fun:mca_pml_ob1_del_procs
+   fun:ompi_mpi_finalize
+   fun:PMPI_Finalize
+   ...
+}
+
+
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Value8
+   fun:opal_thread_compare_exchange_strong_32
+   fun:opal_interval_tree_reader_get_token
+   fun:opal_interval_tree_traverse
+   fun:mca_rcache_base_vma_tree_find_all
+   fun:mca_rcache_base_vma_find_all
+   fun:mca_rcache_rgpusm_finalize
+   fun:mca_rcache_base_module_destroy
+   fun:mca_btl_smcuda_del_procs
+   fun:mca_bml_r2_del_procs
+   fun:mca_pml_ob1_del_procs
+   fun:ompi_mpi_finalize
+   ...
+}
+
+
 {
    <insert_a_suppression_name_here>
    Memcheck:Leak
@@ -307,6 +420,24 @@
    fun:clone
 }
 
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: possible
+   fun:malloc
+   fun:opal_free_list_grow_st
+   obj:*
+   obj:*
+   obj:*
+   obj:*
+   obj:*
+   obj:*
+   fun:PMPI_Send
+   fun:amps_Send
+   fun:amps_BCast
+   fun:amps_SFBCast
+}
+
 ###############################################################
 #
 # Intel F90
@@ -405,3 +536,77 @@
    ...
    fun:main
 }
+
+##############################################################################
+#
+# CUDA
+# These supressions are overly version specific
+#
+##############################################################################
+
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: possible
+   fun:calloc
+   obj:/usr/lib64/libcuda.so.555.42.06
+   ...
+   obj:/usr/lib64/libcuda.so.555.42.06
+}
+
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: possible
+   fun:malloc
+   obj:/usr/lib64/libcuda.so.555.42.06
+   ...
+   obj:/usr/lib64/libcuda.so.555.42.06
+}
+
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: possible
+   fun:malloc
+   obj:/usr/lib64/libcuda.so.555.42.06
+   ...
+   fun:libcudart_static_963dbe518da27ad62a57079a1a0c90814a2710ec
+}
+
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: possible
+   fun:calloc
+   fun:UnknownInlinedFun
+   fun:allocate_dtv
+   fun:_dl_allocate_tls
+   fun:pthread_create@@GLIBC_2.2.5
+   fun:ucs_pthread_create
+   obj:/usr/lib64/libucs.so.0.0.0
+   obj:/usr/lib64/libucs.so.0.0.0
+   fun:ucs_async_set_event_handler
+   obj:/usr/lib64/libucs.so.0.0.0
+   fun:ucs_rcache_create
+   obj:*
+   obj:*
+}
+
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: possible
+   fun:calloc
+   fun:UnknownInlinedFun
+   fun:allocate_dtv
+   fun:_dl_allocate_tls
+   fun:pthread_create@@GLIBC_2.2.5
+   obj:/usr/lib64/libcuda.so.555.42.06
+   obj:/usr/lib64/libcuda.so.555.42.06
+   obj:/usr/lib64/libcuda.so.555.42.06
+   obj:/usr/lib64/libcuda.so.555.42.06
+   obj:/usr/lib64/libcuda.so.555.42.06
+   ...
+}
+
diff --git a/cmake/modules/FindPythonModules.cmake b/cmake/modules/FindPythonModules.cmake
index 2de96cd52..d45212fc2 100644
--- a/cmake/modules/FindPythonModules.cmake
+++ b/cmake/modules/FindPythonModules.cmake
@@ -4,7 +4,7 @@
 function(_find_python_module_internal module_name)
   # Check for presence of the module.  Even though we don't use all the
   # variable names set here, assigning them suppresses their output in CMake.
-  execute_process(COMMAND "${Python3_EXECUTABLE}" -c "import ${module_name}"
+  execute_process(COMMAND "${PARFLOW_PYTHON}" -c "import ${module_name}"
     RESULT_VARIABLE IMPORT_${module_name}_EXITCODE
     OUTPUT_VARIABLE IMPORT_${module_name}_OUTPUT
     ERROR_VARIABLE IMPORT_${module_name}_ERROR
diff --git a/cmake/modules/ParflowTest.cmake b/cmake/modules/ParflowTest.cmake
index 7e8944049..38afe8091 100644
--- a/cmake/modules/ParflowTest.cmake
+++ b/cmake/modules/ParflowTest.cmake
@@ -22,11 +22,11 @@ function (pf_add_parallel_test inputfile topology)
   separate_arguments(targs UNIX_COMMAND ${topology})
   list(APPEND args ${targs})
 
-  add_test (NAME ${testname}_${postfix} COMMAND ${CMAKE_COMMAND} "-DPARFLOW_TEST=${args}" -DMPIEXEC=${MPIEXEC} -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG} "-DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS}" "-DMPIEXEC_POSTFLAGS=${MPIEXEC_POSTFLAGS}" -DPARFLOW_HAVE_OAS3={PARFLOW_HAVE_OAS3} -P ${CMAKE_SOURCE_DIR}/cmake/modules/RunParallelTest.cmake WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+  add_test (NAME ${testname}_${postfix} COMMAND ${CMAKE_COMMAND} "-DPARFLOW_TEST=${args}" -DMPIEXEC=${MPIEXEC} -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG} "-DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS}" "-DMPIEXEC_POSTFLAGS=${MPIEXEC_POSTFLAGS}" -DPARFLOW_HAVE_OAS3={PARFLOW_HAVE_OAS3} -DPARFLOW_HAVE_SILO=${PARFLOW_HAVE_SILO} -DTCL_TCLSH=${TCL_TCLSH} -P ${CMAKE_SOURCE_DIR}/cmake/modules/RunParallelTest.cmake WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
   set_tests_properties(${testname}_${postfix} PROPERTIES TIMEOUT 3600)
 
   if( ${PARFLOW_HAVE_MEMORYCHECK} )
-    add_test (NAME ${testname}_${postfix}_memcheck COMMAND ${CMAKE_COMMAND} -DPARFLOW_HAVE_MEMORYCHECK=${PARFLOW_HAVE_MEMORYCHECK} -DPARFLOW_MEMORYCHECK_COMMAND=${PARFLOW_MEMORYCHECK_COMMAND} -DPARFLOW_MEMORYCHECK_COMMAND_OPTIONS=${PARFLOW_MEMORYCHECK_COMMAND_OPTIONS} "-DPARFLOW_TEST=${args}" -DMPIEXEC=${MPIEXEC} -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG} "-DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS}" "-DMPIEXEC_POSTFLAGS=${MPIEXEC_POSTFLAGS}" -DPARFLOW_HAVE_OAS3=${PARFLOW_HAVE_OAS3} -P ${CMAKE_SOURCE_DIR}/cmake/modules/RunParallelTest.cmake WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+    add_test (NAME ${testname}_${postfix}_memcheck COMMAND ${CMAKE_COMMAND} -DPARFLOW_HAVE_MEMORYCHECK=${PARFLOW_HAVE_MEMORYCHECK} -DPARFLOW_MEMORYCHECK_COMMAND=${PARFLOW_MEMORYCHECK_COMMAND} -DPARFLOW_MEMORYCHECK_COMMAND_OPTIONS=${PARFLOW_MEMORYCHECK_COMMAND_OPTIONS} "-DPARFLOW_TEST=${args}" -DMPIEXEC=${MPIEXEC} -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG} "-DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS}" "-DMPIEXEC_POSTFLAGS=${MPIEXEC_POSTFLAGS}" -DPARFLOW_HAVE_OAS3=${PARFLOW_HAVE_OAS3} -DPARFLOW_HAVE_SILO=${PARFLOW_HAVE_SILO} -DTCL_TCLSH=${TCL_TCLSH} -P ${CMAKE_SOURCE_DIR}/cmake/modules/RunParallelTest.cmake WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
   endif ()
 
 endfunction()
@@ -42,11 +42,11 @@ function (pf_add_sequential_test inputfile)
   list(APPEND args ${inputfile})
   list(APPEND args 1 1 1)
 
-  add_test (NAME ${testname} COMMAND ${CMAKE_COMMAND} "-DPARFLOW_TEST=${args}" -DPARFLOW_HAVE_OAS3=${PARFLOW_HAVE_OAS3} -P ${CMAKE_SOURCE_DIR}/cmake/modules/RunParallelTest.cmake WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+  add_test (NAME ${testname} COMMAND ${CMAKE_COMMAND} "-DPARFLOW_TEST=${args}" -DPARFLOW_HAVE_OAS3=${PARFLOW_HAVE_OAS3} -DPARFLOW_HAVE_SILO=${PARFLOW_HAVE_SILO} -DTCL_TCLSH=${TCL_TCLSH} -P ${CMAKE_SOURCE_DIR}/cmake/modules/RunParallelTest.cmake WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
   set_tests_properties(${testname} PROPERTIES TIMEOUT 3600)
 
   if( ${PARFLOW_HAVE_MEMORYCHECK} )
-    add_test (NAME ${testname}_memcheck COMMAND ${CMAKE_COMMAND} -DPARFLOW_HAVE_MEMORYCHECK=${PARFLOW_HAVE_MEMORYCHECK} -DPARFLOW_MEMORYCHECK_COMMAND=${PARFLOW_MEMORYCHECK_COMMAND} -DPARFLOW_MEMORYCHECK_COMMAND_OPTIONS=${PARFLOW_MEMORYCHECK_COMMAND_OPTIONS} "-DPARFLOW_TEST=${args}" -DMPIEXEC=${MPIEXEC} -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG} "-DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS}" "-DMPIEXEC_POSTFLAGS=${MPIEXEC_POSTFLAGS}" -DPARFLOW_HAVE_OAS3=${PARFLOW_HAVE_OAS3} -P ${CMAKE_SOURCE_DIR}/cmake/modules/RunParallelTest.cmake WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+    add_test (NAME ${testname}_memcheck COMMAND ${CMAKE_COMMAND} -DPARFLOW_HAVE_MEMORYCHECK=${PARFLOW_HAVE_MEMORYCHECK} -DPARFLOW_MEMORYCHECK_COMMAND=${PARFLOW_MEMORYCHECK_COMMAND} -DPARFLOW_MEMORYCHECK_COMMAND_OPTIONS=${PARFLOW_MEMORYCHECK_COMMAND_OPTIONS} "-DPARFLOW_TEST=${args}" -DMPIEXEC=${MPIEXEC} -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG} "-DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS}" "-DMPIEXEC_POSTFLAGS=${MPIEXEC_POSTFLAGS}" -DPARFLOW_HAVE_OAS3=${PARFLOW_HAVE_OAS3} -DPARFLOW_HAVE_SILO=${PARFLOW_HAVE_SILO} -DTCL_TCLSH=${TCL_TCLSH} -P ${CMAKE_SOURCE_DIR}/cmake/modules/RunParallelTest.cmake WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
   endif()
 
 endfunction()
@@ -59,10 +59,10 @@ endfunction()
 function (pf_add_amps_parallel_test test ranks loops)
   set (testname amps-${test}-${ranks}-${loops})
 
-  add_test (NAME ${testname} COMMAND ${CMAKE_COMMAND} -DPARFLOW_TEST=${test} -DPARFLOW_RANKS=${ranks} -DPARFLOW_ARGS=${loops} -DMPIEXEC=${MPIEXEC} -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG} "-DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS}" "-DMPIEXEC_POSTFLAGS=${MPIEXEC_POSTFLAGS}" -DPARFLOW_HAVE_OAS3=${PARFLOW_HAVE_OAS3} -P ${CMAKE_SOURCE_DIR}/cmake/modules/RunAmpsTest.cmake)
+  add_test (NAME ${testname} COMMAND ${CMAKE_COMMAND} -DPARFLOW_TEST=${test} -DPARFLOW_RANKS=${ranks} -DPARFLOW_ARGS=${loops} -DMPIEXEC=${MPIEXEC} -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG} "-DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS}" "-DMPIEXEC_POSTFLAGS=${MPIEXEC_POSTFLAGS}" -DPARFLOW_HAVE_OAS3=${PARFLOW_HAVE_OAS3} -DPARFLOW_HAVE_SILO=${PARFLOW_HAVE_SILO} -DTCL_TCLSH=${TCL_TCLSH} -P ${CMAKE_SOURCE_DIR}/cmake/modules/RunAmpsTest.cmake)
 
   if( ${PARFLOW_HAVE_MEMORYCHECK} )
-    add_test (NAME ${testname}_memcheck COMMAND ${CMAKE_COMMAND} -DPARFLOW_HAVE_MEMORYCHECK=${PARFLOW_HAVE_MEMORYCHECK} -DPARFLOW_MEMORYCHECK_COMMAND=${PARFLOW_MEMORYCHECK_COMMAND} -DPARFLOW_MEMORYCHECK_COMMAND_OPTIONS=${PARFLOW_MEMORYCHECK_COMMAND_OPTIONS} -DPARFLOW_TEST=${test} -DPARFLOW_RANKS=${ranks} -DPARFLOW_ARGS=${loops} -DMPIEXEC=${MPIEXEC} -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG} "-DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS}" "-DMPIEXEC_POSTFLAGS=${MPIEXEC_POSTFLAGS}" -DPARFLOW_HAVE_OAS3=${PARFLOW_HAVE_OAS3} -P ${CMAKE_SOURCE_DIR}/cmake/modules/RunAmpsTest.cmake)
+    add_test (NAME ${testname}_memcheck COMMAND ${CMAKE_COMMAND} -DPARFLOW_HAVE_MEMORYCHECK=${PARFLOW_HAVE_MEMORYCHECK} -DPARFLOW_MEMORYCHECK_COMMAND=${PARFLOW_MEMORYCHECK_COMMAND} -DPARFLOW_MEMORYCHECK_COMMAND_OPTIONS=${PARFLOW_MEMORYCHECK_COMMAND_OPTIONS} -DPARFLOW_TEST=${test} -DPARFLOW_RANKS=${ranks} -DPARFLOW_ARGS=${loops} -DMPIEXEC=${MPIEXEC} -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG} "-DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS}" "-DMPIEXEC_POSTFLAGS=${MPIEXEC_POSTFLAGS}" -DPARFLOW_HAVE_OAS3=${PARFLOW_HAVE_OAS3} -DPARFLOW_HAVE_SILO=${PARFLOW_HAVE_SILO} -DTCL_TCLSH=${TCL_TCLSH} -P ${CMAKE_SOURCE_DIR}/cmake/modules/RunAmpsTest.cmake)
   endif()
 endfunction()
 
diff --git a/cmake/modules/RunAmpsTest.cmake b/cmake/modules/RunAmpsTest.cmake
index 751e715bd..d8636df39 100644
--- a/cmake/modules/RunAmpsTest.cmake
+++ b/cmake/modules/RunAmpsTest.cmake
@@ -9,13 +9,13 @@ cmake_minimum_required(VERSION 3.14)
 # A rank of -1 is used for a sequential run that is run without mpiexec.
 # all parameters passed in as reference
 macro(pf_amps_exec_check cmd ranks args)
-  
+
   set( ENV{PF_TEST} "yes" )
   if (${${ranks}} GREATER 0)
     # Separate potentially space delimited arguments in MPIEXEC_PREFLAGS and MPIEXEC_POSTFLAGS.
     separate_arguments(sep_MPIEXEC_PREFLAGS NATIVE_COMMAND ${MPIEXEC_PREFLAGS})
     separate_arguments(sep_MPIEXEC_POSTFLAGS NATIVE_COMMAND ${MPIEXEC_POSTFLAGS})
-    set( full_command ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${${ranks}} ${sep_MPIEXEC_POSTFLAGS} ${sep_MPIEXEC_PREFLAGS} ${${cmd}} ${${args}} )
+    set( full_command ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${${ranks}} ${sep_MPIEXEC_POSTFLAGS} ${sep_MPIEXEC_PREFLAGS} ./${${cmd}} ${${args}} )
   else()
     set( full_command ./${${cmd}} ${${args}} )
   endif()
diff --git a/cmake/modules/RunParallelTest.cmake b/cmake/modules/RunParallelTest.cmake
index 2bbce985c..a4c353c42 100644
--- a/cmake/modules/RunParallelTest.cmake
+++ b/cmake/modules/RunParallelTest.cmake
@@ -65,7 +65,7 @@ endmacro()
 
 pf_test_clean ()
 
-list(APPEND CMD tclsh ${PARFLOW_TEST})
+list(APPEND CMD ${TCL_TCLSH} ${PARFLOW_TEST})
 
 if (${PARFLOW_HAVE_MEMORYCHECK})
   SET(ENV{PARFLOW_MEMORYCHECK_COMMAND} ${PARFLOW_MEMORYCHECK_COMMAND})
diff --git a/docker/CMakeLists.txt b/docker/CMakeLists.txt
index 44b6aa4cc..275628d1a 100644
--- a/docker/CMakeLists.txt
+++ b/docker/CMakeLists.txt
@@ -71,4 +71,4 @@ add_custom_target(DockerTestRuntime
 #   /PF_SRC/test/python/base_3d/default_single/default_single.py \
 #   -p 1 -q 1 -r 2
 #
-# -----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
\ No newline at end of file
diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile
index 15d747fd2..f8d07cf0b 100644
--- a/docker/dev/Dockerfile
+++ b/docker/dev/Dockerfile
@@ -195,4 +195,4 @@ RUN $CMAKE --install $BASE_PATH/parflow/build --prefix /opt/parflow
 USER ubuntu
 
 ENV PARFLOW_DIR /opt/parflow/
-WORKDIR /opt/parflow/
+WORKDIR /opt/parflow/
\ No newline at end of file
diff --git a/docs/manuals/CMakeLists.txt b/docs/manuals/CMakeLists.txt
index 10859c37b..843fb07b8 100644
--- a/docs/manuals/CMakeLists.txt
+++ b/docs/manuals/CMakeLists.txt
@@ -9,7 +9,7 @@ ADD_LATEX_DOCUMENT(
 
 ADD_LATEX_DOCUMENT(
     dev_manual.tex
-    INPUTS parflow.sty defs.tex dev_manual_intro.tex release_process.tex codeorg.tex documentation.tex modules.tex input.tex equations.tex grid.tex discretize.tex visualization.tex administrative.tex ref.tex ref.grid.tex
+    INPUTS parflow.sty defs.tex dev_manual_intro.tex codeorg.tex documentation.tex modules.tex input.tex equations.tex grid.tex discretize.tex visualization.tex administrative.tex ref.tex ref.grid.tex
     )
 
 ADD_LATEX_DOCUMENT(
diff --git a/docs/manuals/dev_manual.tex b/docs/manuals/dev_manual.tex
index 8a143175d..6d7e9dd4f 100644
--- a/docs/manuals/dev_manual.tex
+++ b/docs/manuals/dev_manual.tex
@@ -61,7 +61,6 @@
 %=============== Include Chapters
 
 \input{dev_manual_intro}
-\input{release_process}
 \input{codeorg}
 \input{documentation}
 \input{modules}
diff --git a/docs/manuals/files.tex b/docs/manuals/files.tex
index bcb8f07c2..d47f07a51 100644
--- a/docs/manuals/files.tex
+++ b/docs/manuals/files.tex
@@ -3598,6 +3598,17 @@ \subsection{Code Parameters}
 pfset Solver.PrintWells False
 \end{verbatim}\end{display}
 
+\pfkey{string}{Solver.PrintReservoirs}{True}
+{
+This key is used to turn on collection and printing of the
+reservoirs data.  The data is collected at intervals given by values
+in the timing information section.  Printing occurs at the
+end of the run when all collected data is written.
+}
+\begin{display}\begin{verbatim}
+pfset Solver.Printeservoirs False
+\end{verbatim}\end{display}
+
 \pfkey{string}{Solver.PrintLSMSink}{False}
 {
 This key is used to turn on printing of the
diff --git a/docs/manuals/release_process.md b/docs/manuals/release_process.md
new file mode 100644
index 000000000..6c42043e8
--- /dev/null
+++ b/docs/manuals/release_process.md
@@ -0,0 +1,124 @@
+# ParFlow elease Process
+
+The release process for ParFlow follows the a standard GitHub
+release process.  The following steps should be followed for creating a new
+ParFlow release.
+
+* Create branch/fork for generating the release
+* Edit the RELEASE-NOTES.md file
+* Regenerate the ParFlow User Manual
+* Commit release and version file changes
+* Create a pull request for the branch/fork
+* Merge the pull request
+ Generate a release on GitHub
+
+## Create branch
+
+Use standard Git/GitHub commands to create a branch for editing some files for the release.
+
+```shell
+VERSION=v3.13.0
+git clone git@github.com:parflow/parflow.git
+git checkout -b $VERSION
+```
+
+## Edit files
+
+Edit the RELEASE-NOTES.md file to add notes about what was changed in
+this release.  Notes should should be appended.  PF has changed to appending
+release changes to the RELEASE notes.
+
+Edit VERSION file with current version.
+
+Edit `./docs/user_Manual/conf.py` and update version number.
+
+### Update pftools version number 
+
+Edit `parflow/pftools/python/setup.py` and increment the version number.
+
+## Regenerate the ParFlowUser Manual
+
+The update should be automated by ReadTheDocs.   Check that a new version has been uploaded.
+  
+## Commit release file changes
+
+Use standard git add and git commit commands to add the modified files
+to the release branch/fork.
+
+```shell
+git add -u :/
+git commit -m "Update release files"
+```
+
+## Create a pull request for the branch/fork
+
+Use GitHub to create a pull request for the release branch.
+  
+## Merge the pull request
+
+Use GitHub to create a pull request for the release branch.
+  
+## Generate a release on GitHub
+
+On the GitHub [Parflow Releases](https://github.com/parflow/parflow/releases)
+page use ``Draft a new release'' to create the release.
+
+Version tag should have format of `vX.Z.Z` version.  Release title
+should have format of `ParFlow Version X.Y.Z`.  The GitHub release
+description can be copied from the release notes markdown file that
+was created in a proir step.
+
+## Generate Docker
+
+```shell
+  docker build -t <hub-user>/<repo-name>[:<tag>]
+  docker push <hub-user>/<repo-name>:<tag>
+```
+
+Example using podman to build and push
+
+```shell
+  VERSION=3.13.0
+  podman build -t docker.io/parflow/parflow:version-$(VERSION) .
+  podman login docker.io
+  podman push docker.io/parflow/parflow:version-$(VERSION)
+  podman tag docker.io/parflow/parflow:version-$(VERSION) docker.io/parflow/parflow:latest
+  podman push  docker.io/parflow/parflow:latest
+```
+
+## Update pftools on PiPy
+
+### Build 
+
+Make sure that Python is enabled through the `PARFLOW_ENABLE_PYTHON` option.
+
+```shell
+mkdir build
+cd build
+cmake .. -D PARFLOW_ENABLE_PYTHON=TRUE
+```
+
+### Create Python Package
+
+Run the following command to create and test a source archive and a wheel
+distribution of the package. Make sure you are running this command in an 
+environment with the `twine` Python package installed.
+
+```shell
+make PythonCreatePackage
+```
+### Publish package
+
+If the distributions pass, run the following command to publish the
+distributions. In order to run this command successfully, you must first set the
+`TWINE_USERNAME` and `TWINE_PASSWORD` environment variables to the username
+and password that you will use to authenticate with PyPI.
+
+```shell
+make PythonPublishPackage
+```
+
+### Check PyPI
+
+Check PyPI to make sure your package update was published correctly.
+
diff --git a/docs/manuals/release_process.tex b/docs/manuals/release_process.tex
deleted file mode 100644
index 4bb965d5d..000000000
--- a/docs/manuals/release_process.tex
+++ /dev/null
@@ -1,81 +0,0 @@
-%=============================================================================
-% Chapter: Release Process
-%=============================================================================
-
-\chapter{Release Process}
-\label{Release Process}
-
-The release process for \parflow{} follows the a standard GitHub
-release process.  The following steps should be followed for creating a new
-\parflow{} release.
-
-\begin{enumerate}
-	\item Create branch/fork for generating the release
-	\item Edit the RELEASE-NOTES.md file
-	\item Regenerate the \parflow{} User Manual
-        \item Commit release file changes
-        \item Create a pull request for the branch/fork
-        \item Merge the pull request
-        \item Generate a release on GitHub
-\end{enumerate}
-
-
-\section{Create branch}
-
-Use standard Git/GitHub commands to create a branch for editing some files for the release.
-
-\section{Edit files}
-
-Edit the RELEASE-NOTES.md file to add notes about what was changed in
-this release.  Old release notes should be deleted, we do not keep a
-running account of release information.
-
-Edit VERSION file with current version.
-
-Edit \code{./docs/user_Manual/conf.py} and update version number.
-
-\section{Regenerate the \parflow{} User Manual}
-
-This should be automated by ReadTheDocs.
-  
-\section{Commit release file changes}
-
-Use standard git add and git commit commands to add the modified files
-to the release branch/fork.
-
-\section{Create a pull request for the branch/fork}
-
-Use GitHub to create a pull request for the release branch.
-  
-\section{Merge the pull request}
-
-Use GitHub to create a pull request for the release branch.
-  
-\section{Generate a release on GitHub}
-
-On the GitHub \code{https://github.com/parflow/parflow/releases} page use
-``Draft a new release'' to create the release.
-
-Version tag should have format of ``vX.Z.Z'' version.  Release title
-should have format of ``ParFlow Version X.Y.Z''.  The GitHub release
-description can be copied from the release notes markdown file that
-was created in a proir step.
-
-\section{Generate Docker}
-
-\begin{display}\begin{verbatim}
-  docker build -t <hub-user>/<repo-name>[:<tag>]
-  docker push <hub-user>/<repo-name>:<tag>
-\end{verbatim}\end{display}
-
-Example using podman to build and push
-
-\begin{display}\begin{verbatim}
-  VERSION=3.12.0
-  podman build -t docker.io/parflow/parflow:version-$(VERSION) .
-  podman login docker.io
-  podman push docker.io/parflow/parflow:version-$(VERSION)
-  podman tag docker.io/parflow/parflow:version-$(VERSION) docker.io/parflow/parflow:latest
-  podman push  docker.io/parflow/parflow:latest
-\end{verbatim}\end{display}
-  
diff --git a/docs/user_manual/CMakeLists.txt b/docs/user_manual/CMakeLists.txt
index 5214fd2da..14c5f9535 100644
--- a/docs/user_manual/CMakeLists.txt
+++ b/docs/user_manual/CMakeLists.txt
@@ -40,23 +40,23 @@ add_custom_target(
 
 include(FindPythonModules)
 find_python_module("sphinx" sphinx_found)
-if (NOT sphinx_found AND NOT PARFLOW_PYTHON_VIRTUAL_ENV)
+if (NOT sphinx_found)
   message(STATUS
-    "Please install Sphinx to generate documentation for ParFlow keys or use the VirtualEnv option.")
+    "Please install Sphinx to generate documentation for ParFlow keys.")
   return()
 endif()
 
 find_python_module("sphinx-rtd-theme" sphinx_theme_found)
-if (NOT sphinx_theme_found AND NOT PARFLOW_PYTHON_VIRTUAL_ENV)
+if (NOT sphinx_theme_found)
   message(STATUS
-    "Please install sphinx-rtd-theme to generate documentation for ParFlow keys or use the VirtualEnv option.")
+    "Please install sphinx-rtd-theme to generate documentation for ParFlow keys.")
   return()
 endif()
 
 find_python_module("sphinxcontrib-bibtex" sphinx_bib_found)
-if (NOT sphinx_bib_found AND NOT PARFLOW_PYTHON_VIRTUAL_ENV)
+if (NOT sphinx_bib_found)
   message(STATUS
-    "Please install sphinxcontrib-bibtex to generate documentation for ParFlow keys or use the VirtualEnv option.")
+    "Please install sphinxcontrib-bibtex to generate documentation for ParFlow keys.")
   return()
 endif()
 
diff --git a/docs/user_manual/Makefile b/docs/user_manual/Makefile
index d4bb2cbb9..2ae441f4a 100644
--- a/docs/user_manual/Makefile
+++ b/docs/user_manual/Makefile
@@ -4,7 +4,7 @@
 # You can set these variables from the command line, and also
 # from the environment for the first two.
 SPHINXOPTS    ?=
-SPHINXBUILD   ?= sphinx-build
+SPHINXBUILD   ?= $(PARFLOW_DIR)/py-env/bin/sphinx-build
 SOURCEDIR     = .
 BUILDDIR      = _build
 
diff --git a/docs/user_manual/conf.py b/docs/user_manual/conf.py
index f7edb3e77..e0d315c94 100644
--- a/docs/user_manual/conf.py
+++ b/docs/user_manual/conf.py
@@ -16,204 +16,209 @@
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
+# sys.path.insert(0, os.path.abspath('.'))
 
 # -- General configuration -----------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
+# needs_sphinx = '1.0'
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.mathjax', 'sphinxcontrib.bibtex']
+extensions = ["sphinx.ext.mathjax", "sphinxcontrib.bibtex"]
 
-bibtex_bibfiles = ['refs.bib']
+bibtex_bibfiles = ["refs.bib"]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['.templates']
+templates_path = [".templates"]
 
 # The suffix of source filenames.
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
-#source_encoding = 'utf-8-sig'
+# source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = u'ParFlow'
-copyright = u'2023'
+project = "ParFlow"
+copyright = "2024"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '3.12.0'
+version = "3.13.0"
 # The full version, including alpha/beta/rc tags.
-release = '3.12.0'
+release = version
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
-#language = None
+# language = None
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-#today = ''
+# today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 exclude_patterns = []
 
 # The reST default role (used for this markup: `text`) to use for all documents.
-#default_role = None
+# default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
+# add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
-#add_module_names = True
+# add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
-#show_authors = False
+# show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+# modindex_common_prefix = []
 
 
 # -- Options for HTML output ---------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-#html_theme_options = {}
+# html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+# html_theme_path = []
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
-#html_title = None
+# html_title = None
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-#html_logo = None
+# html_logo = None
 
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-#html_favicon = None
+# html_favicon = None
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
+
 
 def setup(app):
-    app.add_css_file('css/custom.css')
+    app.add_css_file("css/custom.css")
+
 
 numfig = True
 numfig_secnum_depth = 1
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+# html_last_updated_fmt = '%b %d, %Y'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
-#html_use_smartypants = True
+# html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+# html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-#html_additional_pages = {}
+# html_additional_pages = {}
 
 # If false, no module index is generated.
-#html_domain_indices = True
+# html_domain_indices = True
 
 # If false, no index is generated.
-#html_use_index = True
+# html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
+# html_show_sourcelink = True
 
 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
+# html_show_sphinx = True
 
 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
+# html_show_copyright = True
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
-#html_use_opensearch = ''
+# html_use_opensearch = ''
 
 # This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
+# html_file_suffix = None
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'ParFlowdoc'
+htmlhelp_basename = "ParFlowdoc"
 
 
 # -- Options for LaTeX output --------------------------------------------------
 
 latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
-
-# The font size ('10pt', '11pt' or '12pt').
-#'pointsize': '10pt',
-
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-  ('index', 'ParFlow.tex', u'ParFlow Documentation',
-   u'the ParFlow Developers', 'manual'),
+    (
+        "index",
+        "ParFlow.tex",
+        "ParFlow Documentation",
+        "the ParFlow Developers",
+        "manual",
+    ),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-#latex_logo = None
+# latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
-#latex_use_parts = False
+# latex_use_parts = False
 
 # If true, show page references after internal links.
-#latex_show_pagerefs = False
+# latex_show_pagerefs = False
 
 # If true, show URL addresses after external links.
-#latex_show_urls = False
+# latex_show_urls = False
 
 # Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []
 
 # If false, no module index is generated.
-#latex_domain_indices = True
+# latex_domain_indices = True
 
 
 # -- Options for manual page output --------------------------------------------
@@ -221,12 +226,11 @@ def setup(app):
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    ('index', 'parflow', u'ParFlow Documentation',
-     [u'the ParFlow Developers'], 1)
+    ("index", "parflow", "ParFlow Documentation", ["the ParFlow Developers"], 1)
 ]
 
 # If true, show URL addresses after external links.
-#man_show_urls = False
+# man_show_urls = False
 
 
 # -- Options for Texinfo output ------------------------------------------------
@@ -235,16 +239,22 @@ def setup(app):
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'ParFlow', u'ParFlow Documentation',
-   u'the ParFlow Developers', 'ParFlow', 'An integrated hydrology model.',
-   'Miscellaneous'),
+    (
+        "index",
+        "ParFlow",
+        "ParFlow Documentation",
+        "the ParFlow Developers",
+        "ParFlow",
+        "An integrated hydrology model.",
+        "Miscellaneous",
+    ),
 ]
 
 # Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
+# texinfo_appendices = []
 
 # If false, no module index is generated.
-#texinfo_domain_indices = True
+# texinfo_domain_indices = True
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
+# texinfo_show_urls = 'footnote'
diff --git a/docs/user_manual/index.rst b/docs/user_manual/index.rst
index d3b9cfdef..5e4387356 100644
--- a/docs/user_manual/index.rst
+++ b/docs/user_manual/index.rst
@@ -10,11 +10,12 @@ Welcome to the ParFlow User's Manual
 
 **Reed M. Maxwell** [1]_, **Stefan J. Kollet** [2]_, **Laura E. Condon** [3]_, 
 **Steven G. Smith** [4]_, **Carol S. Woodward** [5]_, **Robert D. Falgout** [6]_, 
-**Ian M. Ferguson** [7]_, **Nicholas Engdahl** [8]_, **Jaro Hokkanen** [9]_, **Basile Hector** [10]_, **James Gilbert** [11]_, **Lindsay Bearup** [7]_, 
+**Ian M. Ferguson** [7]_, **Nicholas Engdahl** [8]_, **Jaro Hokkanen** [9]_, **George Artavanis** [1]_, **Benjamin West** [3]_, **Chen Yang** [1]_, **Basile Hector** [10]_, **James Gilbert** [11]_, **Lindsay Bearup** [7]_, 
 **Jennifer Jefferson** [12]_, **Chuck Baldwin**, **William J. Bosl** [13]_, 
 **Richard Hornung** [4]_, **Steven Ashby** [14]_, **Ketan B. Kulkarni** [15]_
 
-updated: |today|
+| Version: |release|
+| Manual generated : |today|
 
 .. toctree::
    :maxdepth: 2
@@ -91,12 +92,9 @@ updated: |today|
 
 
 | ParFlow is released under the GNU LPGL License
-| Version 1.3, 3 November 2008
-| Copyright © 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc.
-| `http://fsf.org/ <http://fsf.org/>`_.
 | This manual is licensed under the GNU Free Documentation License.
 
-| Copyright © 2019 Reed M. Maxwell, Stefan J. Kollet, Laura E. Condon, 
+| Copyright © 2024 Reed M. Maxwell, Stefan J. Kollet, Laura E. Condon, 
    Ian M. Ferguson, Steven G. Smith, Carol S. Woodward, Nicholas Engdahl. 
    Permission is granted to copy, distribute and/or modify this document 
    under the terms of the GNU Free Documentation License, Version 1.3 or 
diff --git a/docs/user_manual/keys.rst b/docs/user_manual/keys.rst
index 35e0981a8..1ae2d77d4 100644
--- a/docs/user_manual/keys.rst
+++ b/docs/user_manual/keys.rst
@@ -546,6 +546,76 @@ inputs (TCL).
       pfset Geom.domain.Patches           "z-upper x-lower y-lower \
                                             	x-upper y-upper z-lower"
 
+.. _Reservoirs:
+
+Reservoirs
+~~~~~~~~~~
+Here we define reservoirs for the model. Currently reservoirs have only been tested on domains
+where the top of domain lies at the top of the grid. This applies to all box domains and some 
+terrain following grid domains. The format for this section of input
+is:
+
+*string* **Reservoirs.Names** no default This key specifies the names of the
+reservoirs for which input data will be given.
+
+.. container:: list
+
+   ::
+
+      Reservoirs.Names "reservoir_1 reservoir_2 reservoir_3"
+
+*double* **Reservoirs.\ *reservoir_name*.Release_X** no default This key specifies 
+the x location of where the reservoir releases water. This cell will always be placed
+on the domain surface.
+
+*double* **Reservoirs.\ *reservoir_name*.Release_Y** no default This key specifies 
+the y location of where the reservoir releases water. This cell will always be placed
+on the domain surface.
+
+*double* **Reservoirs.\ *reservoir_name*.Intake_X** no default This key specifies 
+the x location of where the reservoir intakes water. This cell will always be placed
+on the domain surface.
+
+*double* **Reservoirs.\ *reservoir_name*.Intake_Y** no default This key specifies 
+the y location of where the reservoir intakes water. This cell will always be placed
+on the domain surface.
+
+.. This value is set as an int because bools do not work with the table reader right now
+*int* **Reservoirs.\ *reservoir_name*.Has_Secondary_Intake_Cell** no default This key specifies if 
+the reservoir has a secondary intake cell, with 0 evaluating to false and 1 evaluating to true. This
+cell will always be placed on the domain surface.
+
+*double* **Reservoirs.\ *reservoir_name*.Secondary_Intake_X** no default This optional key 
+specifies the x location of where the reservoir's secondary intake cell intakes water. This 
+cell will always be placed on the domain surface. This key is only used when the reservoir has
+a secondary intake cell, in which case it is required.
+
+*double* **Reservoirs.\ *reservoir_name*.Secondary_Intake_Y** no default This optional key 
+specifies the y location of where the reservoir's secondary intake cell intakes water. This 
+cell will always be placed on the domain surface. This key is only used when the reservoir has
+a secondary intake cell, in which case it is required.
+
+*double* **Reservoirs.\ *reservoir_name*.Min_Release_Storage** no default This key specifies 
+the storage amount below which the reservoir will stop releasing water. Has units [L\ :sup:`3`].
+
+*double* **Reservoirs.\ *reservoir_name*.Max_Storage** no default This key specifies a reservoirs 
+maximum storage. If storage rises above this value, a reservoir will release extra water if necessary
+to get back down to this amount by the next timestep. Has units [L\ :sup:`3`]
+
+*double* **Reservoirs.\ *reservoir_name*.Storage** no default This key specifies the amount of water 
+stored in the reservoir as a volume. Has same length units as the problem domain i.e. if domain is 
+sized in meters this will be in m\ :sup:`3`.
+
+*double* **Reservoirs.\ *reservoir_name*.Release_Rate** no default [Type: double] This key specifies 
+the rate in volume/time [L\ :sup:`3` \ :sup:`-1`] that the reservoir release water. The amount of time over which 
+this amount is released is independent of solver timestep size.
+
+Overland_Flow_Solver
+
+*string* **Reservoirs.Overland_Flow_Solver** no default This key specifies which overland flow 
+condition is used in the domain so that the slopes aroundthe reservoirs can be adjusted properly. 
+Supported Options are **OverlandFlow** and **OverlandKinematic**.
+
 .. _Timing Information:
 
 Timing Information
@@ -589,9 +659,7 @@ result for every “real time” cycle interval length needed.
 indicate the time step number that will be associated with the first
 advection cycle in a transient problem. The value **-1** indicates that
 advection is not to be done. The value **0** indicates that advection
-should begin with the given initial conditions. Values greater than
-**0** are intended to mean “restart” from some previous “checkpoint”
-time-step, but this has not yet been implemented.
+should begin with the given initial conditions. 
 
 .. container:: list
 
@@ -2148,6 +2216,165 @@ Example of setting :math:`x` and :math:`y` slopes by file:
       pfset TopoSlopesY.GeomNames "domain"
       pfset TopoSlopesY.FileName "lw.1km.slope_y.pfb"
 
+
+.. _Channelwidths:
+
+Channelwidths
+~~~~~~~~~~~~~
+These keys are in development. They have been added to the pftools
+Python interface and can be set to read and print channewidth values,
+but have not yet been integrated with overland flow.
+
+Here, channel width values are assigned to the upper boundary
+of the domain using one of the methods described below.
+
+The format for this section of input is:
+
+*list* **Solver.Nonlinear.ChannelWidthExistX** False This key specifies
+whether a channelwidthX input is provided.
+
+*list* **Solver.Nonlinear.ChannelWidthExistY** False This key specifies
+whether a channelwidthY input is provided.
+
+*list* **ChannelWidthX.GeomNames** no default This key specifies all of
+the geometries on which a different ChannelWidthX values
+will be assigned. ChannelWidthX may be assigned by **PFBFile** or **NCFile**
+as **Constant** by geometry. These geometries must cover the entire
+upper surface of the computational domain.
+
+.. container:: list
+
+   ::
+
+      pfset ChannelWidthX.GeomNames       "domain"      ## TCL syntax
+
+      <runname>.ChannelWidthX.GeomNames = "domain"      ## Python syntax
+
+*list* **ChannelWidthY.GeomNames** no default This key specifies all of
+the geometries on which a different ChannelWidthY values
+will be assigned. ChannelWidthX may be assigned by **PFBFile** or **NCFile**
+as **Constant** by geometry. These geometries must cover the entire
+upper surface of the computational domain.
+
+.. container:: list
+
+   ::
+
+      pfset ChannelWidthY.GeomNames       "domain"      ## TCL syntax
+
+      <runname>.ChannelWidthY.GeomNames = "domain"      ## Python syntax
+
+
+*string* **ChannelWidthX.Type** Constant This key specifies which method
+is to be used to assign ChannelWidthX. The choices currently
+available are **Constant** which indicates that a constant is to be
+assigned to all grid cells within a geometry, **PFBFile** which
+indicates that all values are read in from a distributed, grid-based
+ParFlow 2D binary file and **NCFile** which indicates that all values
+are read in from a netcdf file.
+
+.. container:: list
+
+   ::
+
+      pfset ChannelWidthX.Type "Constant"         ## TCL syntax
+
+      <runname>.ChannelWidthX.Type = "Constant"   ## Python syntax
+
+*double* **ChannelWidthX.Geom.\ *geometry_name*.Value** 0.0 This key
+specifies the value assigned to all points in the named geometry,
+*geometry_name*, if the type was set to constant.
+
+.. container:: list
+
+   ::
+
+      pfset ChannelWidthX.Geom.domain.Value       100     ## TCL syntax
+
+      <runname>.ChannelWidthX.Geom.domain.Value = 100     ## Python syntax
+
+*double* **ChannelWidthX.FileName** no default This key specifies the
+value assigned to all points be read in from a ParFlow 2D binary file or
+a netcdf file.
+
+.. container:: list
+
+   ::
+
+      pfset ChannelWidthX.FileName       "channel_x.pfb"    ## TCL syntax
+
+      <runname>.ChannelWidthX.FileName = "channel_x.pfb"    ## Python syntax
+
+*string* **ChannelWidthY.Type** Constant This key specifies which method
+is to be used to assign ChannelWidthY. The choices currently
+available are **Constant** which indicates that a constant is to be
+assigned to all grid cells within a geometry, **PFBFile** which
+indicates that all values are read in from a distributed, grid-based
+ParFlow 2D binary file and **NCFile** which indicates that all values
+are read in from a netcdf file.
+
+.. container:: list
+
+   ::
+
+      pfset ChannelWidthY.Type "Constant"         ## TCL syntax
+
+      <runname>.ChannelWidthY.Type = "Constant"   ## Python syntax
+
+*double* **ChannelWidthY.Geom.\ *geometry_name*.Value** 0.0 This key
+specifies the value assigned to all points in the named geometry,
+*geometry_name*, if the type was set to constant.
+
+.. container:: list
+
+   ::
+
+      pfset ChannelWidthY.Geom.domain.Value       100     ## TCL syntax
+
+      <runname>.ChannelWidthY.Geom.domain.Value = 100     ## Python syntax
+
+*double* **ChannelWidthY.FileName** no default This key specifies the
+value assigned to all points be read in from a ParFlow 2D binary file or
+a netcdf file.
+
+.. container:: list
+
+   ::
+
+      pfset ChannelWidthY.FileName       "channel_y.pfb"    ## TCL syntax
+
+      <runname>.ChannelWidthY.FileName = "channel_y.pfb"    ## Python syntax
+
+
+Example of setting :math:`x` and :math:`y` channelwidths by geometry:
+
+.. container:: list
+
+   ::
+
+      pfset ChannelWidthX.Type "Constant"
+      pfset ChannelWidthX.GeomNames "domain"
+      pfset ChannelWidthX.Geom.domain.Value 100
+
+      pfset ChannelWidthY.Type "Constant"
+      pfset ChannelWidthY.GeomNames "domain"
+      pfset ChannelWidthY.Geom.domain.Value 100
+
+Example of setting :math:`x` and :math:`y` channelwidths by file:
+
+.. container:: list
+
+   ::
+
+      pfset ChannelWidthX.Type "PFBFile"
+      pfset ChannelWidthX.GeomNames "domain"
+      pfset ChannelWidthX.FileName "channel_x.pfb"
+
+      pfset ChannelWidthY.Type "PFBFile"
+      pfset ChannelWidthY.GeomNames "domain"
+      pfset ChannelWidthY.FileName "channel_y.pfb"
+
+
 .. _Retardation:
 
 Retardation
@@ -4227,6 +4454,26 @@ minimum value for the :math:`\bar{S_{f}}` used in the
    <runname>.Solver.OverlandKinematic.Epsilon = 1E-7     ## Python syntax
 
 
+*string* **Solver.PrintInitialConditions** True This key is used to
+      turn on printing of the initial conditions.  This includes the
+      pressure, saturation, slopes, etc.  By default the initial
+      conditions output is generated before the first time
+      advancement; when doing a restart this leads to a duplication
+      of files on each restart.  Setting this key to False will
+      prevent the duplication.
+
+      Note setting this key to False overrides the other individual
+      output flags that are enabled.
+
+.. container:: list
+
+   ::
+
+      pfset Solver.PrintInitialConditions False           ## TCL syntax
+
+      <runname>.Solver.PrintInitalConditions = False     ## Python syntax
+
+
 *string* **Solver.PrintSubsurf** True This key is used to turn on
 printing of the subsurface data, Permeability and Porosity. The data is
 printed after it is generated and before the main time stepping loop -
@@ -4240,6 +4487,19 @@ only once during the run. The data is written as a ParFlow binary file.
 
       <runname>.Solver.PrintSubsurf = False     ## Python syntax
 
+*string* **Solver.PrintChannelWidth** True This key is used to turn on
+printing of the channelwidth data, ChannelWidthX and ChannelWidthY. The data
+is printed before the main time stepping loop - only once during the run.
+The data is written as two ParFlow binary files.
+
+.. container:: list
+
+   ::
+
+      pfset Solver.PrintChannelWidth False           ## TCL syntax
+
+      <runname>.Solver.PrintChannelWidth = False     ## Python syntax      
+
 *string* **Solver.PrintPressure** True This key is used to turn on
 printing of the pressure data. The printing of the data is controlled by
 values in the timing information section. The data is written as a PFB
@@ -4325,6 +4585,19 @@ occurs at the end of the run when all collected data is written.
 
       <runname>.Solver.PrintWells = False    ## Python syntax
 
+*string* **Solver.PrintReservoirs** True This key is used to turn on
+collection and printing of the reservoir data. The data is collected at
+intervals given by values in the timing information section. Printing
+occurs at the end of the run when all collected data is written.
+
+.. container:: list
+
+   ::
+
+      pfset Solver.PrintReservoirs False          ## TCL syntax
+
+      <runname>.Solver.PrintReservoirs = False    ## Python syntax
+
 *string* **Solver.PrintLSMSink** False This key is used to turn on
 printing of the flux array passed from ``CLM`` to ParFlow. 
 Printing occurs at each **DumpInterval** time.
@@ -4935,7 +5208,7 @@ help with slope errors and issues and provides some diagnostic information.  The
       <runname>.Solver.ResetSurfacePressure.ResetPressure  = 0.0    ## Python syntax
 
 
-*logical* **Solver.SurfacePredictor** False This key activates a routine that uses the evap trans flux and available water storage in a surface cell to predict whether an unsaturated cell will pond during the next timestep. The pressure values are set with the key below.
+*logical* **Solver.SurfacePredictor** False This key activates a routine that uses the evap trans flux, Darcy flux, and available water storage in a surface cell to predict whether an unsaturated cell will pond during the next timestep. The pressure values are set with the key below.
 .. container:: list
 
    ::
@@ -4943,7 +5216,7 @@ help with slope errors and issues and provides some diagnostic information.  The
       pfset Solver.SurfacePredictor        True        ## TCL syntax
       <runname>.Solver.SurfacePredictor  = "True"    ## Python syntax
 
-*double* **Solver.SurfacePredictor.PressureValue** 0.00001 This key specifies a surface pressure if the **SurfacePredictor** key above is True and ponded conditions are predicted at a surface cell.
+*double* **Solver.SurfacePredictor.PressureValue** 0.00001 This key specifies a surface pressure if the **SurfacePredictor** key above is True and ponded conditions are predicted at a surface cell.  A negative value allows the surface predictor algorithm to esimate the new surface pressure based on surrounding fluxes.
 
 .. container:: list
 
@@ -6006,7 +6279,7 @@ lon) or (time,z, y, x)*
       	x = 200 ;
       	y = 200 ;
       	z = 40 ;
-      	time = UNLIMITED ; // (1 currently)
+      	time = UNLIMITED ; 
       variables:
       	double time(time) ;
       	double pressure(time, z, y, x) ;
@@ -6018,15 +6291,23 @@ initial conditions.**
 *string* **ICPressure.Type** no default This key sets flag for initial
 conditions to be read from a NetCDF file.
 
+NetCDF4 files may have more than one timestep in the file.   By default the first
+timestep will be read.  The TimeStep attribute is used to specify the timestep
+to be used for the initial pressure.   Negative values are allowed to index
+from the end.   "-1" is often a useful index to read the last timestep
+in the file.
+
 .. container:: list
 
    ::
 
       pfset ICPressure.Type   "NCFile"        ## TCL syntax  
       pfset Geom.domain.ICPressure.FileName "initial_condition.nc" ## TCL syntax
+      pfset Geom.domain.ICPressure.TimeStep -1 ## TCL syntax
 
       <runname>.ICPressure.Type = "NCFile"    ## Python syntax
       <runname>.Geom.domain.ICPressure.FileName = "initial_condition.nc" ## Python syntax
+      <runname>.Geom.domain.ICPressure.TimeStep = -1 ## Python syntax
 
 NetCDF4 Slopes
 ~~~~~~~~~~~~~~
diff --git a/examples/.gitignore b/examples/.gitignore
index f730756cd..a81786cba 100644
--- a/examples/.gitignore
+++ b/examples/.gitignore
@@ -1,6 +1,7 @@
-*.csv
-*.out.*
-*.output.*
-*.pfidb
-*.pfb.dist
-*_output.*
+/*.csv
+/*.out.*
+/*.output.*
+/*.pfidb
+/*.pfb.dist
+/*_output.*
+/test_output
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 6e503ee7d..a6d296885 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -1,7 +1,7 @@
 
 include(ParflowTest)
 
-set(TESTS bc.00.tcl cond.tcl default_richards.tcl default_single.tcl forsyth2.tcl forsyth5.tcl impes.internalbc.tcl impes.simple.tcl llnl.tcl llnl-perm.tcl test_X3Y2PlusSinXYPlus1.tcl test_XPlusYPlusZ.tcl test_X.tcl test_XYZTPlus1.tcl well_00.tcl well_01.tcl)
+set(TESTS bc.00.tcl cond.tcl example_richards.tcl example_single.tcl forsyth2.tcl forsyth5.tcl impes.internalbc.tcl impes.simple.tcl llnl.tcl llnl-perm.tcl test_X3Y2PlusSinXYPlus1.tcl test_XPlusYPlusZ.tcl test_X.tcl test_XYZTPlus1.tcl well_00.tcl well_01.tcl)
 
 if((${PARFLOW_HAVE_HYPRE}) AND (${PARFLOW_HAVE_SILO}))
   list(APPEND TESTS impes.plinear.tcl)
@@ -12,6 +12,11 @@ if((${PARFLOW_HAVE_CUDA}) OR (${PARFLOW_HAVE_KOKKOS}) OR (${PARFLOW_HAVE_OMP}))
   list(REMOVE_ITEM TESTS test_XPlusYPlusZ.tcl)
 endif()
 
+# Remove this failing test
+if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+  list(REMOVE_ITEM TESTS test_XPlusYPlusZ.tcl)
+endif()
+
 if((${PARFLOW_AMPS_LAYER} STREQUAL "mpi1") OR (${PARFLOW_AMPS_LAYER} STREQUAL "cuda"))
   set(PARALLEL_TESTS richards.plinear.tcl)
 endif()
@@ -20,7 +25,7 @@ if(${PARFLOW_HAVE_OAS3})
   # Exclude tests that use Richards solver
   list(REMOVE_ITEM TESTS 
     cond.tcl
-    default_richards.tcl
+    example_richards.tcl
     forsyth2.tcl
     forsyth5.tcl
     test_X3Y2PlusSinXYPlus1.tcl
@@ -36,19 +41,24 @@ foreach(inputfile ${PARALLEL_TESTS})
   pf_add_parallel_test(${inputfile} "2 1 1")
 endforeach()
 
-install(FILES default_single.tcl pftest.tcl DESTINATION examples)
+install(FILES example_single.tcl example_single.py pftest.tcl DESTINATION examples)
 install(FILES
-  correct_output/default_single.out.concen.0.00.00000.pfsb
-  correct_output/default_single.out.concen.0.00.00001.pfsb
-  correct_output/default_single.out.concen.0.00.00002.pfsb
-  correct_output/default_single.out.concen.0.00.00003.pfsb
-  correct_output/default_single.out.concen.0.00.00004.pfsb
-  correct_output/default_single.out.concen.0.00.00005.pfsb
-  correct_output/default_single.out.perm_x.pfb
-  correct_output/default_single.out.perm_y.pfb
-  correct_output/default_single.out.perm_z.pfb
-  correct_output/default_single.out.press.00000.pfb
+  correct_output/example_single.out.concen.0.00.00000.pfsb
+  correct_output/example_single.out.concen.0.00.00001.pfsb
+  correct_output/example_single.out.concen.0.00.00002.pfsb
+  correct_output/example_single.out.concen.0.00.00003.pfsb
+  correct_output/example_single.out.concen.0.00.00004.pfsb
+  correct_output/example_single.out.concen.0.00.00005.pfsb
+  correct_output/example_single.out.perm_x.pfb
+  correct_output/example_single.out.perm_y.pfb
+  correct_output/example_single.out.perm_z.pfb
+  correct_output/example_single.out.press.00000.pfb
   DESTINATION examples/correct_output)
 
+if (PARFLOW_ENABLE_PYTHON)
+  set(PY_TESTS example_single)
 
-
+  foreach(inputfile ${PY_TESTS})
+    pf_add_py_test(${inputfile})
+  endforeach()
+endif()
diff --git a/examples/correct_output/default_richards.out.perm_x.pfb b/examples/correct_output/example_richards.out.perm_x.pfb
similarity index 100%
rename from examples/correct_output/default_richards.out.perm_x.pfb
rename to examples/correct_output/example_richards.out.perm_x.pfb
diff --git a/examples/correct_output/default_richards.out.perm_y.pfb b/examples/correct_output/example_richards.out.perm_y.pfb
similarity index 100%
rename from examples/correct_output/default_richards.out.perm_y.pfb
rename to examples/correct_output/example_richards.out.perm_y.pfb
diff --git a/examples/correct_output/default_richards.out.perm_z.pfb b/examples/correct_output/example_richards.out.perm_z.pfb
similarity index 100%
rename from examples/correct_output/default_richards.out.perm_z.pfb
rename to examples/correct_output/example_richards.out.perm_z.pfb
diff --git a/examples/correct_output/default_richards.out.porosity.pfb b/examples/correct_output/example_richards.out.porosity.pfb
similarity index 100%
rename from examples/correct_output/default_richards.out.porosity.pfb
rename to examples/correct_output/example_richards.out.porosity.pfb
diff --git a/examples/correct_output/default_richards.out.press.00001.pfb b/examples/correct_output/example_richards.out.press.00001.pfb
similarity index 100%
rename from examples/correct_output/default_richards.out.press.00001.pfb
rename to examples/correct_output/example_richards.out.press.00001.pfb
diff --git a/examples/correct_output/default_richards.out.specific_storage.pfb b/examples/correct_output/example_richards.out.specific_storage.pfb
similarity index 100%
rename from examples/correct_output/default_richards.out.specific_storage.pfb
rename to examples/correct_output/example_richards.out.specific_storage.pfb
diff --git a/examples/correct_output/default_single.out.concen.0.00.00000.pfsb b/examples/correct_output/example_single.out.concen.0.00.00000.pfsb
similarity index 100%
rename from examples/correct_output/default_single.out.concen.0.00.00000.pfsb
rename to examples/correct_output/example_single.out.concen.0.00.00000.pfsb
diff --git a/examples/correct_output/default_single.out.concen.0.00.00001.pfsb b/examples/correct_output/example_single.out.concen.0.00.00001.pfsb
similarity index 100%
rename from examples/correct_output/default_single.out.concen.0.00.00001.pfsb
rename to examples/correct_output/example_single.out.concen.0.00.00001.pfsb
diff --git a/examples/correct_output/default_single.out.concen.0.00.00002.pfsb b/examples/correct_output/example_single.out.concen.0.00.00002.pfsb
similarity index 100%
rename from examples/correct_output/default_single.out.concen.0.00.00002.pfsb
rename to examples/correct_output/example_single.out.concen.0.00.00002.pfsb
diff --git a/examples/correct_output/default_single.out.concen.0.00.00003.pfsb b/examples/correct_output/example_single.out.concen.0.00.00003.pfsb
similarity index 100%
rename from examples/correct_output/default_single.out.concen.0.00.00003.pfsb
rename to examples/correct_output/example_single.out.concen.0.00.00003.pfsb
diff --git a/examples/correct_output/default_single.out.concen.0.00.00004.pfsb b/examples/correct_output/example_single.out.concen.0.00.00004.pfsb
similarity index 100%
rename from examples/correct_output/default_single.out.concen.0.00.00004.pfsb
rename to examples/correct_output/example_single.out.concen.0.00.00004.pfsb
diff --git a/examples/correct_output/default_single.out.concen.0.00.00005.pfsb b/examples/correct_output/example_single.out.concen.0.00.00005.pfsb
similarity index 100%
rename from examples/correct_output/default_single.out.concen.0.00.00005.pfsb
rename to examples/correct_output/example_single.out.concen.0.00.00005.pfsb
diff --git a/examples/correct_output/default_single.out.perm_x.pfb b/examples/correct_output/example_single.out.perm_x.pfb
similarity index 100%
rename from examples/correct_output/default_single.out.perm_x.pfb
rename to examples/correct_output/example_single.out.perm_x.pfb
diff --git a/examples/correct_output/default_single.out.perm_y.pfb b/examples/correct_output/example_single.out.perm_y.pfb
similarity index 100%
rename from examples/correct_output/default_single.out.perm_y.pfb
rename to examples/correct_output/example_single.out.perm_y.pfb
diff --git a/examples/correct_output/default_single.out.perm_z.pfb b/examples/correct_output/example_single.out.perm_z.pfb
similarity index 100%
rename from examples/correct_output/default_single.out.perm_z.pfb
rename to examples/correct_output/example_single.out.perm_z.pfb
diff --git a/examples/correct_output/default_single.out.press.00000.pfb b/examples/correct_output/example_single.out.press.00000.pfb
similarity index 100%
rename from examples/correct_output/default_single.out.press.00000.pfb
rename to examples/correct_output/example_single.out.press.00000.pfb
diff --git a/examples/default_richards.tcl b/examples/example_richards.tcl
similarity index 93%
rename from examples/default_richards.tcl
rename to examples/example_richards.tcl
index 7ff7d6550..44c6ff629 100644
--- a/examples/default_richards.tcl
+++ b/examples/example_richards.tcl
@@ -11,9 +11,45 @@ namespace import Parflow::*
 
 pfset FileVersion 4
 
-pfset Process.Topology.P 1
-pfset Process.Topology.Q 1
-pfset Process.Topology.R 1
+#-----------------------------------------------------------------------------
+# Process command line arguments
+#-----------------------------------------------------------------------------
+if { [info exists ::env(PF_TEST) ] } {
+    set running_as_test 1
+} else {
+    set running_as_test 0
+}
+
+set arglen [llength $argv]
+set index 0
+set parsed_argv {}
+
+while {$index < $arglen} {
+    set arg [lindex $argv $index]
+    switch -exact $arg {
+        -t {
+            set running_as_test 1
+        }
+	default  {
+	    lappend parsed_argv $arg
+	}
+    }
+    incr index
+}
+
+#-----------------------------------------------------------------------------
+# Process Topology
+#-----------------------------------------------------------------------------
+
+if  { [llength $parsed_argv] == 3 } {
+    pfset Process.Topology.P        [lindex $parsed_argv 0]
+    pfset Process.Topology.Q        [lindex $parsed_argv 1]
+    pfset Process.Topology.R        [lindex $parsed_argv 2]
+} else {
+    pfset Process.Topology.P        1
+    pfset Process.Topology.Q        1
+    pfset Process.Topology.R        1
+}
 
 #---------------------------------------------------------
 # Computational Grid
@@ -401,16 +437,18 @@ pfset Geom.domain.SpecificStorage.Value 1.0e-4
 #-----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
 #-----------------------------------------------------------------------------
-pfrun default_richards
-pfundist default_richards
+
+set TEST example_richards
+
+pfrun $TEST
+pfundist $TEST
 
 #-----------------------------------------------------------------------------
 # If running as test; check output.
 # You do not need this for normal PF input files; this is done so the examples
 # are run and checked as part of our testing process.
 #-----------------------------------------------------------------------------
-if { [info exists ::env(PF_TEST) ] } {
-    set TEST default_richards
+if { $running_as_test } {
     source pftest.tcl
     set sig_digits 4
 
diff --git a/examples/example_single.py b/examples/example_single.py
new file mode 100644
index 000000000..c75d49576
--- /dev/null
+++ b/examples/example_single.py
@@ -0,0 +1,478 @@
+#
+# Import the ParFlow TCL package
+#
+from parflow import Run
+from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file, pf_test_file_with_abs
+import sys
+
+
+run_name = "example_single"
+
+example_single = Run(run_name, __file__)
+
+correct_output_dir_name = get_absolute_path("correct_output")
+
+# -----------------------------------------------------------------------------
+# File input version number
+# -----------------------------------------------------------------------------
+example_single.FileVersion = 4
+
+# -----------------------------------------------------------------------------
+# Process Topology
+# -----------------------------------------------------------------------------
+
+example_single.Process.Topology.P = 1
+example_single.Process.Topology.Q = 1
+example_single.Process.Topology.R = 1
+
+# -----------------------------------------------------------------------------
+# Computational Grid
+# -----------------------------------------------------------------------------
+example_single.ComputationalGrid.Lower.X = -10.0
+example_single.ComputationalGrid.Lower.Y = 10.0
+example_single.ComputationalGrid.Lower.Z = 1.0
+
+example_single.ComputationalGrid.DX = 8.8888888888888893
+example_single.ComputationalGrid.DY = 10.666666666666666
+example_single.ComputationalGrid.DZ = 1.0
+
+example_single.ComputationalGrid.NX = 18
+example_single.ComputationalGrid.NY = 15
+example_single.ComputationalGrid.NZ = 8
+
+# -----------------------------------------------------------------------------
+# The Names of the GeomInputs
+# -----------------------------------------------------------------------------
+example_single.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
+
+# -----------------------------------------------------------------------------
+# Domain Geometry Input
+# -----------------------------------------------------------------------------
+example_single.GeomInput.domain_input.InputType = "Box"
+example_single.GeomInput.domain_input.GeomName = "domain"
+
+# -----------------------------------------------------------------------------
+# Domain Geometry
+# -----------------------------------------------------------------------------
+example_single.Geom.domain.Lower.X = -10.0
+example_single.Geom.domain.Lower.Y = 10.0
+example_single.Geom.domain.Lower.Z = 1.0
+
+example_single.Geom.domain.Upper.X = 150.0
+example_single.Geom.domain.Upper.Y = 170.0
+example_single.Geom.domain.Upper.Z = 9.0
+
+example_single.Geom.domain.Patches = "left right front back bottom top"
+
+# -----------------------------------------------------------------------------
+# Background Geometry Input
+# -----------------------------------------------------------------------------
+example_single.GeomInput.background_input.InputType = "Box"
+example_single.GeomInput.background_input.GeomName = "background"
+
+# -----------------------------------------------------------------------------
+# Background Geometry
+# -----------------------------------------------------------------------------
+example_single.Geom.background.Lower.X = -99999999.0
+example_single.Geom.background.Lower.Y = -99999999.0
+example_single.Geom.background.Lower.Z = -99999999.0
+
+example_single.Geom.background.Upper.X = 99999999.0
+example_single.Geom.background.Upper.Y = 99999999.0
+example_single.Geom.background.Upper.Z = 99999999.0
+
+# -----------------------------------------------------------------------------
+# Source_Region Geometry Input
+# -----------------------------------------------------------------------------
+example_single.GeomInput.source_region_input.InputType = "Box"
+example_single.GeomInput.source_region_input.GeomName = "source_region"
+
+# -----------------------------------------------------------------------------
+# Source_Region Geometry
+# -----------------------------------------------------------------------------
+example_single.Geom.source_region.Lower.X = 65.56
+example_single.Geom.source_region.Lower.Y = 79.34
+example_single.Geom.source_region.Lower.Z = 4.5
+
+example_single.Geom.source_region.Upper.X = 74.44
+example_single.Geom.source_region.Upper.Y = 89.99
+example_single.Geom.source_region.Upper.Z = 5.5
+
+
+# -----------------------------------------------------------------------------
+# Concen_Region Geometry Input
+# -----------------------------------------------------------------------------
+example_single.GeomInput.concen_region_input.InputType = "Box"
+example_single.GeomInput.concen_region_input.GeomName = "concen_region"
+
+# -----------------------------------------------------------------------------
+# Concen_Region Geometry
+# -----------------------------------------------------------------------------
+example_single.Geom.concen_region.Lower.X = 60.0
+example_single.Geom.concen_region.Lower.Y = 80.0
+example_single.Geom.concen_region.Lower.Z = 4.0
+
+example_single.Geom.concen_region.Upper.X = 80.0
+example_single.Geom.concen_region.Upper.Y = 100.0
+example_single.Geom.concen_region.Upper.Z = 6.0
+
+# -----------------------------------------------------------------------------
+# Perm
+# -----------------------------------------------------------------------------
+example_single.Geom.Perm.Names = "background"
+
+example_single.Geom.background.Perm.Type = "Constant"
+example_single.Geom.background.Perm.Value = 4.0
+
+example_single.Perm.TensorType = "TensorByGeom"
+
+example_single.Geom.Perm.TensorByGeom.Names = "background"
+
+example_single.Geom.background.Perm.TensorValX = 1.0
+example_single.Geom.background.Perm.TensorValY = 1.0
+example_single.Geom.background.Perm.TensorValZ = 1.0
+
+# -----------------------------------------------------------------------------
+# Phases
+# -----------------------------------------------------------------------------
+
+example_single.Phase.Names = "water"
+
+example_single.Phase.water.Density.Type = "Constant"
+example_single.Phase.water.Density.Value = 1.0
+
+example_single.Phase.water.Viscosity.Type = "Constant"
+example_single.Phase.water.Viscosity.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Contaminants
+# -----------------------------------------------------------------------------
+example_single.Contaminants.Names = "tce"
+example_single.Contaminants.tce.Degradation.Value = 0.0
+
+# -----------------------------------------------------------------------------
+# Gravity
+# -----------------------------------------------------------------------------
+
+example_single.Gravity = 1.0
+
+# -----------------------------------------------------------------------------
+# Setup timing info
+# -----------------------------------------------------------------------------
+
+example_single.TimingInfo.BaseUnit = 1.0
+example_single.TimingInfo.StartCount = 0
+example_single.TimingInfo.StartTime = 0.0
+example_single.TimingInfo.StopTime = 1000.0
+example_single.TimingInfo.DumpInterval = -1
+
+# -----------------------------------------------------------------------------
+# Porosity
+# -----------------------------------------------------------------------------
+
+example_single.Geom.Porosity.GeomNames = "background"
+
+example_single.Geom.background.Porosity.Type = "Constant"
+example_single.Geom.background.Porosity.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Domain
+# -----------------------------------------------------------------------------
+example_single.Domain.GeomName = "domain"
+
+# -----------------------------------------------------------------------------
+# Mobility
+# -----------------------------------------------------------------------------
+example_single.Phase.water.Mobility.Type = "Constant"
+example_single.Phase.water.Mobility.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Retardation
+# -----------------------------------------------------------------------------
+example_single.Geom.Retardation.GeomNames = "background"
+example_single.Geom.background.tce.Retardation.Type = "Linear"
+example_single.Geom.background.tce.Retardation.Rate = 0.0
+
+# -----------------------------------------------------------------------------
+# Time Cycles
+# -----------------------------------------------------------------------------
+example_single.Cycle.Names = "constant"
+example_single.Cycle.constant.Names = "alltime"
+example_single.Cycle.constant.alltime.Length = 1
+example_single.Cycle.constant.Repeat = -1
+
+# -----------------------------------------------------------------------------
+# Wells
+# -----------------------------------------------------------------------------
+example_single.Wells.Names = "snoopy"
+
+example_single.Wells.snoopy.InputType = "Recirc"
+
+example_single.Wells.snoopy.Cycle = "constant"
+
+example_single.Wells.snoopy.ExtractionType = "Flux"
+example_single.Wells.snoopy.InjectionType = "Flux"
+
+example_single.Wells.snoopy.X = 71.0
+example_single.Wells.snoopy.Y = 90.0
+example_single.Wells.snoopy.ExtractionZLower = 5.0
+example_single.Wells.snoopy.ExtractionZUpper = 5.0
+example_single.Wells.snoopy.InjectionZLower = 2.0
+example_single.Wells.snoopy.InjectionZUpper = 2.0
+
+example_single.Wells.snoopy.ExtractionMethod = "Standard"
+example_single.Wells.snoopy.InjectionMethod = "Standard"
+
+example_single.Wells.snoopy.alltime.Extraction.Flux.water.Value = 5.0
+example_single.Wells.snoopy.alltime.Injection.Flux.water.Value = 7.5
+example_single.Wells.snoopy.alltime.Injection.Concentration.water.tce.Fraction = 0.1
+
+# -----------------------------------------------------------------------------
+# Boundary Conditions: Pressure
+# -----------------------------------------------------------------------------
+example_single.BCPressure.PatchNames = "left right front back bottom top"
+
+example_single.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+example_single.Patch.left.BCPressure.Cycle = "constant"
+example_single.Patch.left.BCPressure.RefGeom = "domain"
+example_single.Patch.left.BCPressure.RefPatch = "bottom"
+example_single.Patch.left.BCPressure.alltime.Value = 14.0
+
+example_single.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+example_single.Patch.right.BCPressure.Cycle = "constant"
+example_single.Patch.right.BCPressure.RefGeom = "domain"
+example_single.Patch.right.BCPressure.RefPatch = "bottom"
+example_single.Patch.right.BCPressure.alltime.Value = 9.0
+
+example_single.Patch.front.BCPressure.Type = "FluxConst"
+example_single.Patch.front.BCPressure.Cycle = "constant"
+example_single.Patch.front.BCPressure.alltime.Value = 0.0
+
+example_single.Patch.back.BCPressure.Type = "FluxConst"
+example_single.Patch.back.BCPressure.Cycle = "constant"
+example_single.Patch.back.BCPressure.alltime.Value = 0.0
+
+example_single.Patch.bottom.BCPressure.Type = "FluxConst"
+example_single.Patch.bottom.BCPressure.Cycle = "constant"
+example_single.Patch.bottom.BCPressure.alltime.Value = 0.0
+
+example_single.Patch.top.BCPressure.Type = "FluxConst"
+example_single.Patch.top.BCPressure.Cycle = "constant"
+example_single.Patch.top.BCPressure.alltime.Value = 0.0
+
+# #-----------------------------------------------------------------------------
+# # Boundary Conditions: Temperature
+# #-----------------------------------------------------------------------------
+# example_single.BCTemperature.PatchNames = 'left right front back bottom top'
+# #
+# example_single.Patch.left.BCTemperature.Type = 'DirConst'
+# example_single.Patch.left.BCTemperature.Cycle = 'constant'
+# example_single.Patch.left.BCTemperature.alltime.Value = 288.15
+# #
+# example_single.Patch.right.BCTemperature.Type = 'DirConst'
+# example_single.Patch.right.BCTemperature.Cycle = 'constant'
+# example_single.Patch.right.BCTemperature.alltime.Value = 295.
+# example_single.Patch.right.BCTemperature.alltime.Value = 293.15
+# #
+# example_single.Patch.front.BCTemperature.Type = 'DirConst'
+# example_single.Patch.front.BCTemperature.Type = 'FluxConst'
+# example_single.Patch.front.BCTemperature.Cycle = 'constant'
+# example_single.Patch.front.BCTemperature.alltime.Value = 305.
+# example_single.Patch.front.BCTemperature.alltime.Value = 0.0
+# #
+# example_single.Patch.back.BCTemperature.Type = 'DirConst'
+# example_single.Patch.back.BCTemperature.Type = 'FluxConst'
+# example_single.Patch.back.BCTemperature.Cycle = 'constant'
+# example_single.Patch.back.BCTemperature.alltime.Value = 295.
+# example_single.Patch.back.BCTemperature.alltime.Value = 0.0
+# #
+# example_single.Patch.bottom.BCTemperature.Type = 'FluxConst'
+# example_single.Patch.bottom.BCTemperature.Cycle = 'constant'
+# example_single.Patch.bottom.BCTemperature.alltime.Value = 0.0
+# #
+# example_single.Patch.top.BCTemperature.Type = 'FluxConst'
+# example_single.Patch.top.BCTemperature.Cycle = 'constant'
+# example_single.Patch.top.BCTemperature.alltime.Value = 0.0
+
+# -----------------------------------------------------------------------------
+# Phase sources:
+# -----------------------------------------------------------------------------
+
+example_single.PhaseSources.water.Type = "Constant"
+example_single.PhaseSources.water.GeomNames = "background"
+example_single.PhaseSources.water.Geom.background.Value = 0.0
+
+example_single.PhaseSources.Type = "Constant"
+example_single.PhaseSources.GeomNames = "background"
+example_single.PhaseSources.Geom.background.Value = 0.0
+
+example_single.PhaseConcen.water.tce.Type = "Constant"
+example_single.PhaseConcen.water.tce.GeomNames = "concen_region"
+example_single.PhaseConcen.water.tce.Geom.concen_region.Value = 0.8
+
+# #-----------------------------------------------------------------------------
+# # Temperature sources:
+# #-----------------------------------------------------------------------------
+# example_single.TempSources.Type = 'Constant'
+# example_single.TempSources.GeomNames = 'background'
+# example_single.TempSources.Geom.background.Value = 0.0
+
+# -----------------------------------------------------------------------------
+# Specific Storage
+# -----------------------------------------------------------------------------
+
+example_single.SpecificStorage.Type = "Constant"
+example_single.SpecificStorage.GeomNames = "background"
+example_single.Geom.background.SpecificStorage.Value = 1.0e-5
+
+# -----------------------------------------------------------------------------
+# Heat Capacity
+# -----------------------------------------------------------------------------
+
+example_single.Phase.water.HeatCapacity.Type = "Constant"
+example_single.Phase.water.HeatCapacity.GeomNames = "background"
+example_single.Phase.water.Geom.background.HeatCapacity.Value = 4000.0
+
+# ---------------------------------------------------------
+# Topo slopes in x-direction
+# ---------------------------------------------------------
+
+example_single.TopoSlopesX.Type = "Constant"
+example_single.TopoSlopesX.GeomNames = "domain"
+example_single.TopoSlopesX.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Topo slopes in y-direction
+# ---------------------------------------------------------
+
+example_single.TopoSlopesY.Type = "Constant"
+example_single.TopoSlopesY.GeomNames = "domain"
+example_single.TopoSlopesY.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
+
+example_single.Mannings.Type = "Constant"
+example_single.Mannings.GeomNames = "domain"
+example_single.Mannings.Geom.domain.Value = 2.3e-7
+
+
+# -----------------------------------------------------------------------------
+# The Solver Impes MaxIter default value changed so to get previous
+# results we need to set it back to what it was
+# -----------------------------------------------------------------------------
+example_single.Solver.MaxIter = 5
+
+# -----------------------------------------------------------------------------
+# Run and Unload the ParFlow output files
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/example_single")
+mkdir(new_output_dir_name)
+example_single.run(working_directory=new_output_dir_name)
+
+# -----------------------------------------------------------------------------
+# Check results for regression
+# -----------------------------------------------------------------------------
+
+passed = True
+
+i = 0
+timestep = str(i).rjust(5, "0")
+
+sig_digits = 4
+abs_value = 1e-12
+
+test_files = ["press"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {filename}",
+        sig_digits,
+        abs_value,
+    ):
+        passed = False
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {filename}",
+        sig_digits,
+        abs_value,
+    ):
+        passed = False
+
+# This doesn't work yet, test does not work for pfsb files.
+# test_files = ["concen.0.00"]
+# for i in range(0,5):
+#     timestep = str(i).rjust(5, '0')
+#     for test_file in test_files:
+#         filename = f"/{run_name}.out.{test_file}.{timestep}.pfsb"
+#         if not pf_test_file(new_output_dir_name + filename, correct_output_dir_name + filename, f"Max difference in {test_file}", sig_digits):
+#             passed = False
+
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
+
+
+# -----------------------------------------------------------------------------
+# Run and Unload the ParFlow output files
+# -----------------------------------------------------------------------------
+# pfrun example_single
+# pfundist example_single
+
+# -----------------------------------------------------------------------------
+# If running as test; check output.
+# You do not need this for normal PF input files; this is done so the examples
+# are run and checked as part of our testing process.
+# -----------------------------------------------------------------------------
+# if { [info exists ::env(PF_TEST) ] } {
+#     set TEST example_single
+#     source pftest.tcl
+#     set sig_digits 4
+
+#     set passed 1
+
+#     #
+#     # Tests
+#     #
+#     if ![pftestFile $TEST.out.press.00000.pfb "Max difference in Pressure" $sig_digits] {
+# 	set passed 0
+#     }
+
+#     if ![pftestFile $TEST.out.perm_x.pfb "Max difference in perm_x" $sig_digits] {
+# 	set passed 0
+#     }
+#     if ![pftestFile $TEST.out.perm_y.pfb "Max difference in perm_y" $sig_digits] {
+# 	set passed 0
+#     }
+#     if ![pftestFile $TEST.out.perm_z.pfb "Max difference in perm_z" $sig_digits] {
+# 	set passed 0
+#     }
+#
+#     foreach i "00000 00001 00002 00003 00004 00005" {
+# 	if ![pftestFile $TEST.out.concen.0.00.$i.pfsb "Max difference in concen timestep $i" $sig_digits] {
+# 	    set passed 0
+# 	}
+#     }
+
+#     if $passed {
+# 	puts "$TEST : PASSED"
+#     } {
+# 	puts "$TEST : FAILED"
+#     }
+# }
+# example_single.run()
diff --git a/examples/default_single.tcl b/examples/example_single.tcl
similarity index 94%
rename from examples/default_single.tcl
rename to examples/example_single.tcl
index 4e8bee34c..7d579693d 100644
--- a/examples/default_single.tcl
+++ b/examples/example_single.tcl
@@ -10,13 +10,45 @@ namespace import Parflow::*
 #-----------------------------------------------------------------------------
 pfset FileVersion 4
 
+#-----------------------------------------------------------------------------
+# Process command line arguments
+#-----------------------------------------------------------------------------
+if { [info exists ::env(PF_TEST) ] } {
+    set running_as_test 1
+} else {
+    set running_as_test 0
+}
+
+set arglen [llength $argv]
+set index 0
+set parsed_argv {}
+
+while {$index < $arglen} {
+    set arg [lindex $argv $index]
+    switch -exact $arg {
+        -t {
+            set running_as_test 1
+        }
+	default  {
+	    lappend parsed_argv $arg
+	}
+    }
+    incr index
+}
+
 #-----------------------------------------------------------------------------
 # Process Topology
 #-----------------------------------------------------------------------------
 
-pfset Process.Topology.P        1
-pfset Process.Topology.Q        1
-pfset Process.Topology.R        1
+if  { [llength $parsed_argv] == 3 } {
+    pfset Process.Topology.P        [lindex $parsed_argv 0]
+    pfset Process.Topology.Q        [lindex $parsed_argv 1]
+    pfset Process.Topology.R        [lindex $parsed_argv 2]
+} else {
+    pfset Process.Topology.P        1
+    pfset Process.Topology.Q        1
+    pfset Process.Topology.R        1
+}
 
 #-----------------------------------------------------------------------------
 # Computational Grid
@@ -360,16 +392,18 @@ pfset Solver.MaxIter 5
 #-----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
 #-----------------------------------------------------------------------------
-pfrun default_single
-pfundist default_single
+set TEST example_single
+
+pfrun $TEST
+pfundist $TEST
 
 #-----------------------------------------------------------------------------
 # If running as test; check output.
 # You do not need this for normal PF input files; this is done so the examples
 # are run and checked as part of our testing process.
 #-----------------------------------------------------------------------------
-if { [info exists ::env(PF_TEST) ] } {
-    set TEST default_single
+
+if { $running_as_test } {
     source pftest.tcl
     set sig_digits 4
 
diff --git a/examples/forsyth5.tcl b/examples/forsyth5.tcl
index 1233fcd54..5db83443f 100644
--- a/examples/forsyth5.tcl
+++ b/examples/forsyth5.tcl
@@ -1,11 +1,30 @@
-#BHEADER***********************************************************************
-# (c) 1997   The Regents of the University of California
+#BHEADER**********************************************************************
 #
-# See the file COPYRIGHT_and_DISCLAIMER for a complete copyright
-# notice, contact person, and disclaimer.
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+#  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+#  by the Parflow Team (see the CONTRIBUTORS file)
+#  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
 #
-# $Revision: 1.1.1.1 $
-#EHEADER***********************************************************************
+#  This file is part of Parflow. For details, see
+#  http://www.llnl.gov/casc/parflow
+#
+#  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+#  for the GNU Lesser General Public License.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License (as published
+#  by the Free Software Foundation) version 2.1 dated February 1999.
+#
+#  This program is distributed in the hope that it will be useful, but
+#  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+#  and conditions of the GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+#  USA
+#**********************************************************************EHEADER
 
 ###############################################################################
 #
diff --git a/misc/bin/replace_headers_c.pl b/misc/bin/replace_headers_c.pl
index 5d74b5182..6b68f4fa4 100755
--- a/misc/bin/replace_headers_c.pl
+++ b/misc/bin/replace_headers_c.pl
@@ -1,7 +1,7 @@
 #! /usr/bin/perl
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow\@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
@@ -37,31 +37,31 @@
 
 $cpytxt = <<END;
 BHEADER**********************************************************************
-
-  Copyright (c) 1995-2009, Lawrence Livermore National Security,
-  LLC. Produced at the Lawrence Livermore National Laboratory. Written
-  by the Parflow Team (see the CONTRIBUTORS file)
-  <parflow\@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
-
-  This file is part of Parflow. For details, see
-  http://www.llnl.gov/casc/parflow
-
-  Please read the COPYRIGHT file or Our Notice and the LICENSE file
-  for the GNU Lesser General Public License.
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License (as published
-  by the Free Software Foundation) version 2.1 dated February 1999.
-
-  This program is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
-  and conditions of the GNU General Public License for more details.
-
-  You should have received a copy of the GNU Lesser General Public
-  License along with this program; if not, write to the Free Software
-  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-  USA
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow\@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
 **********************************************************************EHEADER
 END
 
@@ -133,7 +133,7 @@ sub selectHFiles {
     # Do substitutions
     #
 #    $str =~ s!/*BHEADER(.*)EHEADER*/!!sgm;
-    $str =~ s!BHEADER(.*)EHEADER!$cpytxt!sgm;
+    $str =~ s!BHEADER(.*?)EHEADER!$cpytxt!sgm;
 
     #
     # Write new file
diff --git a/misc/bin/replace_headers_f.pl b/misc/bin/replace_headers_f.pl
index f242e68bb..a0a3d23da 100755
--- a/misc/bin/replace_headers_f.pl
+++ b/misc/bin/replace_headers_f.pl
@@ -1,7 +1,7 @@
 #! /usr/bin/perl
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow\@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
@@ -38,7 +38,7 @@
 $cpytxt = <<END;
 C BHEADER**********************************************************************
 C 
-C   Copyright (c) 1995-2009, Lawrence Livermore National Security,
+C   Copyright (c) 1995-2024, Lawrence Livermore National Security,
 C   LLC. Produced at the Lawrence Livermore National Laboratory. Written
 C   by the Parflow Team (see the CONTRIBUTORS file)
 C   <parflow\@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
@@ -133,7 +133,7 @@ sub selectHFiles {
     # Do substitutions
     #
 #    $str =~ s!/*BHEADER(.*)EHEADER*/!!sgm;
-    $str =~ s!cBHEADER(.*)EHEADER\**!$cpytxt!sgm;
+    $str =~ s!cBHEADER(.*?)EHEADER\**!$cpytxt!sgm;
 
     #
     # Write new file
diff --git a/misc/bin/replace_headers_shell.pl b/misc/bin/replace_headers_shell.pl
index 1524cda89..50c0b04dc 100755
--- a/misc/bin/replace_headers_shell.pl
+++ b/misc/bin/replace_headers_shell.pl
@@ -1,7 +1,7 @@
 #! /usr/bin/perl
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow\@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
@@ -38,7 +38,7 @@
 $cpytxt = <<END;
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow\@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
@@ -132,7 +132,7 @@ sub selectHFiles {
     #
     # Do substitutions
     #
-    $str =~ s!#BHEADER(.*)EHEADER\**!$cpytxt!sgm;
+    $str =~ s!#BHEADER(.*?)EHEADER\**!$cpytxt!sgm;
 
     #
     # Write new file
diff --git a/pf-keys/CMakeLists.txt b/pf-keys/CMakeLists.txt
index 0704025b7..ddcece7f3 100644
--- a/pf-keys/CMakeLists.txt
+++ b/pf-keys/CMakeLists.txt
@@ -1,16 +1,4 @@
 
-# -----------------------------------------------------------------------------
-# Make sure our Python has the module that we need
-# -----------------------------------------------------------------------------
-
-include(FindPythonModules)
-find_python_module("yaml" yaml_found)
-if (NOT yaml_found AND NOT PARFLOW_PYTHON_VIRTUAL_ENV)
-  message(STATUS
-    "Please install yaml to generate ParFlow keys for Python module or use the VirtualEnv option.")
-  return()
-endif()
-
 # -----------------------------------------------------------------------------
 # Build generated python file for key definitions
 # -----------------------------------------------------------------------------
diff --git a/pf-keys/definitions/geom.yaml b/pf-keys/definitions/geom.yaml
index f6d1f6b79..3f63f44de 100644
--- a/pf-keys/definitions/geom.yaml
+++ b/pf-keys/definitions/geom.yaml
@@ -30,6 +30,14 @@ Domain:
         type: ChildHandler
         class_name: TopoSlopesYItem
         location: /TopoSlopesY/Geom
+      ChannelWidthXUpdater:
+        type: ChildHandler
+        class_name: ChannelWidthXItem
+        location: /ChannelWidthX/Geom
+      ChannelWidthYUpdater:
+        type: ChildHandler
+        class_name: ChannelWidthYItem
+        location: /ChannelWidthY/Geom
       ManningsGeomUpdater:
         type: ChildHandler
         class_name: ManningsGeomItem
@@ -813,6 +821,7 @@ Geom:
           EnumDomain:
             enum_list:
               - Constant
+              - PFBFile
 
       Value:
         help: >
@@ -1240,6 +1249,12 @@ Geom:
           AnyString:
           ValidFile:
 
+      TimeStep:
+        help: >
+          This key specifies the timestep to be read from the pressure file.   Currently this key is only valid for NetCDF files.
+        domains:
+          IntValue:
+
       RefElevation:
         help: >
           [Type: double] This key specifies the reference elevation on which the reference pressure is given for type HydroStaticDepth
@@ -1479,6 +1494,146 @@ TopoSlopesY:
         domains:
           DoubleValue:
 
+# -----------------------------------------------------------------------------
+# ChannelWidth
+# -----------------------------------------------------------------------------
+
+ChannelWidth:
+  __doc__: >
+    Setting filename for channel width data 
+  Width:
+    __doc__: >
+      Setting filename for channel width data
+    FileName:
+      help: >
+        [Type: string] This key is the name of the PFB file that contains channel width data which ParFlow uses.
+      domains:
+        Anystring:
+        ValidFile:
+
+ChannelWidthX:
+  __doc__: >
+    Setting data for channel width in the X direction
+  GeomNames:
+    help: >
+      [Type: string] This key specifies all of the geometries on which different channel width x values will be assigned. Channel widths
+      may be assigned by PFBFile or as Constant by geometry. These geometries must cover the entire upper surface of the
+      computational domain. 
+    domains:
+      EnumDomain:
+        locations: 
+          - /GeomInput/{GeomInputItem}/GeomNames
+          - /GeomInput/{GeomInputItem}/GeomName
+    handlers:
+      ChannelWidthXUpdater:
+        type: ChildrenHandler
+        class_name: ChannelWidthXItem
+        location: ./Geom
+
+  Type:
+    help: >
+      [Type: string] This key specifies which method is to be used to assign channel widths. The choices currently available are
+      Constant which indicates that a constant is to be assigned to all grid cells within a geometry and PFBFile which
+      indicates that all values are read in from a distributed, grid-based ParFlow binary file. If NetCDF is used, NCFile can be
+      specified, which will read in channel widths from a NetCDF file.
+    default: Constant
+    domains:
+      EnumDomain:
+        enum_list:
+          - Constant
+          - PFBFile
+          - NCFile
+
+  FileName:
+    help: >
+      [Type: string] This key specifies the value assigned to all points be read in from a ParFlow binary file.
+    domains:
+      AnyString:
+      ValidFile:
+
+  Geom:
+    __class__: ChannelWidthXGeom
+    __rst__:
+      skip:
+    __doc__: ''
+    .{geom_name}:
+      __class__: ChannelWidthXItem
+      __rst__:
+        name: ChannelWidthX.Geom.{geom_name}
+      __doc__: >
+        Setting value for channel widths in the X direction
+      __simput__:
+        type: VariableTable
+
+      Value:
+        help: >
+          [Type: double] This key specifies the value assigned to all points in the named geometry, geometry_name, if the type was set
+          to constant.
+        default: 0.0
+        domains:
+          DoubleValue:
+
+ChannelWidthY:
+  __doc__: >
+    Setting data for channel width in the Y direction
+  GeomNames:
+    help: >
+      [Type: string] This key specifies all of the geometries on which a different y channel width values will be assigned. Channel width values
+      may be assigned by PFBFile or as Constant by geometry. These geometries must cover the entire upper surface of the
+      computational domain.
+    domains:
+      EnumDomain:
+        locations: 
+          - /GeomInput/{GeomInputItem}/GeomNames
+          - /GeomInput/{GeomInputItem}/GeomName
+    handlers:
+      ChannelWidthYUpdater:
+        type: ChildrenHandler
+        class_name: ChannelWidthYItem
+        location: ./Geom
+
+  Type:
+    help: >
+      [Type: string] This key specifies which method is to be used to assign channel widths. The choices currently available are
+      Constant which indicates that a constant is to be assigned to all grid cells within a geometry and PFBFile which
+      indicates that all values are read in from a distributed, grid-based ParFlow binary file. If NetCDF is used, NCFile can be
+      specified, which will read in channel width from a NetCDF file.
+    default: Constant
+    domains:
+      EnumDomain:
+        enum_list:
+          - Constant
+          - PFBFile
+          - NCFile
+
+  FileName:
+    help: >
+      [Type: string] This key specifies the value assigned to all points be read in from a ParFlow binary file.
+    domains:
+      AnyString:
+      ValidFile:
+
+  Geom:
+    __class__: ChannelWidthYGeom
+    __rst__:
+      skip:
+    __doc__: ''
+    .{geom_name}:
+      __class__: ChannelWidthYItem
+      __rst__:
+        name: ChannelWidthY.Geom.{geom_name}
+      __doc__: >
+        Setting value for slopes in the Y direction
+      __simput__:
+        type: VariableTable
+      Value:
+        help: >
+          [Type: double] This key specifies the value assigned to all points in the named geometry, geometry_name, if the type was set
+          to constant.
+        default: 0.0
+        domains:
+          DoubleValue:
+
 # -----------------------------------------------------------------------------
 # Capillary Pressure
 # -----------------------------------------------------------------------------
@@ -1627,3 +1782,4 @@ FBz:
       EnumDomain:
         enum_list:
           - PFBFile
+
diff --git a/pf-keys/definitions/reservoirs.yaml b/pf-keys/definitions/reservoirs.yaml
new file mode 100644
index 000000000..1c63b8b46
--- /dev/null
+++ b/pf-keys/definitions/reservoirs.yaml
@@ -0,0 +1,124 @@
+# reservoirs.yaml
+# Definitions of all reservoir items
+
+# -----------------------------------------------------------------------------
+# Reservoirs
+# -----------------------------------------------------------------------------
+
+Reservoirs:
+  __doc__: >
+    Here we define the reservoirs for the model.
+
+  # Reservoirs.Names not in grep
+  Names:
+    help: >
+      [Type: string] This key specifies the names for the reservoirs for which input data will be given.
+    default: ""
+    domains:
+      AnyString:
+    handlers:
+      ReservoirUpdater:
+        type: ChildrenHandler
+        class_name: ReservoirItem
+
+  .{reservoir_name}:
+    __class__: ReservoirItem
+    __rst__:
+      name: Reservoirs.{reservoir_name}
+    __doc__: >
+      Specifying properties for reservoirs
+    __simput__:
+      type: VariableTable
+      table_label: Reservoir
+
+    Release_X:
+      help: >
+        [Type: double] This key specifies the x location of where the reservoir releases water. This 
+        cell will always be placed on the domain surface.
+      domains:
+        DoubleValue:
+
+    Release_Y:
+      help: >
+        [Type: double] This key specifies the y location of where the reservoir releases water. This 
+        cell will always be placed on the domain surface.
+      domains:
+        DoubleValue:
+
+    Intake_X:
+      help: >
+        [Type: double] This key specifies the x location of where the reservoir intakes water
+      domains:
+        DoubleValue:
+
+    Intake_Y:
+
+      help: >
+        [Type: double] This key specifies the y location of where the reservoir intakes water.
+      domains:
+        DoubleValue:
+
+    Secondary_Intake_X:
+
+      help: >
+        [Type: double] This key specifies the x location of the optional secondary intake cell. This 
+        cell will always be placed on the domain surface.
+      default: -1.0
+      domains:
+        DoubleValue:
+
+    Secondary_Intake_Y:
+
+      help: >
+        [Type: double] This key specifies the y location of the optional secondary intake cell. This 
+        cell will always be placed on the domain surface.
+      default: -1.0
+      domains:
+        DoubleValue:
+
+    Has_Secondary_Intake_Cell:
+    # This value is set as an int because bools do not work with the table reader right now
+      help: >
+        [Type: bool] This key specifies if the reservoir has a secondary intake cell, 
+        with 0 evaluating to false and 1 evaluating to true. This cell will always be 
+        placed on the domain surface.
+      default: False
+      domains:
+        IntValue:
+
+    Min_Release_Storage:
+
+      help: >
+        [Type: double] This key specifies the storage amount below which the reservoir will stop releasing water
+      domains:
+        DoubleValue:
+
+    Max_Storage:
+
+      help: >
+        [Type: double] This key specifies a reservoir's maximum storage. If storage rises above this value, a 
+        reservoir will release extra water if necessary to get back down to this amount by the next timestep.
+      domains:
+        DoubleValue:
+
+    Storage:
+
+      help: >
+        [Type: double] This key specifies the amount of water stored in the reservoir as a volume. Has same length
+        units as the problem domain i.e. if domain is sized in meters this will be in m^3
+      domains:
+        DoubleValue:
+
+    Release_Rate:
+      help: >
+        [Type: double] This key specifies the rate in volume/time [L^3 T^-1] that the reservoir release water.
+        The amount of time over which this amount is released is independent of solver timestep size
+      domains:
+        DoubleValue:
+
+  Overland_Flow_Solver:
+    help: >
+      [Type: string] This key specifies which overland flow condition is used in the domain so that the slopes around
+      the reservoirs can be adjusted properly. Supported Options are OverlandFlow and OverlandKinematic.
+    domains:
+      AnyString:
diff --git a/pf-keys/definitions/run.yaml b/pf-keys/definitions/run.yaml
index 9f2529a26..2c3b8bfa8 100644
--- a/pf-keys/definitions/run.yaml
+++ b/pf-keys/definitions/run.yaml
@@ -21,8 +21,11 @@ BaseRun:
     - Contaminants
     - Domain
     - Wells
+    - Reservoirs
     - TopoSlopesX
     - TopoSlopesY
+    - ChannelWidthX
+    - ChannelWidthY
     - Mannings
     - ICPressure
     - InternalBC
diff --git a/pf-keys/definitions/solver.yaml b/pf-keys/definitions/solver.yaml
index 97de14e41..5edb90faf 100644
--- a/pf-keys/definitions/solver.yaml
+++ b/pf-keys/definitions/solver.yaml
@@ -1731,6 +1731,16 @@ Solver:
     domains:
       BoolDomain:
 
+  PrintInitialConditions:
+    help: >
+      [Type: boolean/string] This key is used to turn on printing of the initial conditions.   This includes the pressure, saturation,
+      slopes, etc.   By default the initial conditions output is generated before the first time advancement.   When doing a restart
+      this leads to a duplication of files on each restart.   Setting this key to False will prevent the duplication.
+    default: True
+    domains:
+      BoolDomain:
+
+
   # missing from manual
   PrintMannings:
     help: >
@@ -1801,6 +1811,15 @@ Solver:
     domains:
       BoolDomain:
 
+  # missing from manual
+  PrintChannelWidth:
+    help: >
+      [Type: boolean/string] This key is used to turn on printing of the saturation data. The printing of the data is controlled by values in the
+      timing information section. The data is written as a PFB file.
+    default: True 
+    domains:
+      BoolDomain:
+
   # missing from manual
   PrintSpecificStorage:
     help: >
@@ -1853,6 +1872,14 @@ Solver:
     default: True
     domains:
       BoolDomain:
+  
+  PrintReservoirs:
+    help: >
+      [Type: boolean/string] This key is used to turn on collection and printing of the reservoir data. The data is collected at intervals given by
+      values in the timing information section. Printing occurs at the end of the run when all collected data is written.
+    default: True
+    domains:
+      BoolDomain:
 
   # WritePfbSlopes is missing from the manual.
   WritePfbSlopes:
@@ -2193,6 +2220,21 @@ Solver:
       domains:
         BoolDomain:
 
+    ChannelWidthExistX:
+      help: >
+        [Type: boolean/string] This key specifies whether channel width is used to be used in the X direction, default is false. The default
+        indicates a false or ChannelWidthX value of 0 everywhere in the domain.
+      default: False
+      domains:
+        BoolDomain:
+    ChannelWidthExistY:
+      help: >
+        [Type: boolean/string] This key specifies whether channel width is used to be used in the Y direction, default is false. The default
+        indicates a false or ChannelWidthY value of 0 everywhere in the domain.
+      default: False
+      domains:
+        BoolDomain:
+
     ResidualTol:
       help: >
         [Type: double] This key specifies the tolerance that measures how much the relative reduction in the nonlinear residual should
diff --git a/pf-keys/definitions/timing.yaml b/pf-keys/definitions/timing.yaml
index 871f899ca..307c9c0ee 100644
--- a/pf-keys/definitions/timing.yaml
+++ b/pf-keys/definitions/timing.yaml
@@ -35,7 +35,6 @@ TimingInfo:
     domains:
       IntValue:
         min_value: -1
-        max_value: 0
 
   StartTime:
     help: >
diff --git a/pf-keys/definitions/wells.yaml b/pf-keys/definitions/wells.yaml
index 7bd8e30fd..f37f00139 100644
--- a/pf-keys/definitions/wells.yaml
+++ b/pf-keys/definitions/wells.yaml
@@ -12,7 +12,8 @@ Wells:
   # Wells.Names not in grep
   Names:
     help: >
-      [Type: string] This key specifies the names fo the wells for which input data will be given.
+      [Type: string] This key specifies the names for the wells for which input data will be given.
+    default: ""
     domains:
       AnyString:
     handlers:
diff --git a/pf-keys/generators/pf-python.py b/pf-keys/generators/pf-python.py
index 215752661..40fda8da1 100644
--- a/pf-keys/generators/pf-python.py
+++ b/pf-keys/generators/pf-python.py
@@ -1,8 +1,8 @@
-r'''
+r"""
 This module provides the infrastructure to load and generate the ParFlow
 database structure as Python classes so IDE and runtime environment can
 be used to query the help and constraints associated to each key.
-'''
+"""
 
 from datetime import datetime
 import json
@@ -13,87 +13,95 @@
 # -----------------------------------------------------------------------------
 
 YAML_MODULES_TO_PROCESS = [
-    'core',
-    'geom',
-    'solver',
-    'wells',
-    'timing',
-    'phase',
-    'bconditions',
-    'netcdf',
-    'metadata',
-    'run'
+    "core",
+    "geom",
+    "solver",
+    "wells",
+    "reservoirs",
+    "timing",
+    "phase",
+    "bconditions",
+    "netcdf",
+    "metadata",
+    "run",
 ]
 
 
 # -----------------------------------------------------------------------------
 
+
 def is_field(key, definition):
-    if key.startswith('_'):
+    if key.startswith("_"):
         return False
 
     value = definition[key]
 
-    if '__doc__' in value:
+    if "__doc__" in value:
         return False
 
-    return any(x in value for x in ['help', '__field__'])
+    return any(x in value for x in ["help", "__field__"])
 
 
 # -----------------------------------------------------------------------------
 
+
 def is_class(key, definition):
-    if key[0] in ['_', '.']:
+    if key[0] in ["_", "."]:
         return False
 
     value = definition[key]
 
-    if '__doc__' in value:
+    if "__doc__" in value:
         return True
 
-    return not any(x in value for x in ['help', '__field__'])
+    return not any(x in value for x in ["help", "__field__"])
 
 
 # -----------------------------------------------------------------------------
 
+
 def has_value(key, definition):
-    if key.startswith('_'):
+    if key.startswith("_"):
         return False
 
-    return '__value__' in definition
+    return "__value__" in definition
 
 
 # -----------------------------------------------------------------------------
 
+
 def has_prefix(key, definition):
-    if key.startswith('_'):
+    if key.startswith("_"):
         return False
 
-    return '__prefix__' in definition
+    return "__prefix__" in definition
 
 
 # -----------------------------------------------------------------------------
 
+
 def is_class_item(key, definition):
-    return key.startswith('.')
+    return key.startswith(".")
 
 
 # -----------------------------------------------------------------------------
 
+
 def is_dynamic(key, definition):
     item = definition[key]
-    return '__class__' in item and '__from__' in item
+    return "__class__" in item and "__from__" in item
 
 
 # -----------------------------------------------------------------------------
 
+
 def json_to_python(txt):
     replacements = [
-        (' true,', ' True,'),
-        (' false,', ' False,'),
-        (' null', ' None'),
-        (': true', ': True'),
-        (': false', ': False')
+        (" true,", " True,"),
+        (" false,", " False,"),
+        (" null", " None"),
+        (": true", ": True"),
+        (": false", ": False"),
     ]
     for r in replacements:
         txt = txt.replace(*r)
@@ -102,6 +110,7 @@ def json_to_python(txt):
 
 # -----------------------------------------------------------------------------
 
+
 def yaml_value(yval):
     if isinstance(yval, str):
         try:
@@ -114,11 +123,13 @@ def yaml_value(yval):
 
 # -----------------------------------------------------------------------------
 
+
 class ValidationSummary:
-    '''
+    """
     This is the class to generate the Python library of ParFlow keys from the
     yaml files.
-    '''
+    """
+
     def __init__(self):
         self.has_duplicate = False
         self.class_name_count = {}
@@ -132,38 +143,42 @@ def add_class(self, class_name):
         return self.class_name_count[class_name] - 1
 
     def get_deduplicate_class_name(self, class_name, class_definition=None):
-        if class_definition and '__class__' in class_definition:
-            return class_definition['__class__']
+        if class_definition and "__class__" in class_definition:
+            return class_definition["__class__"]
         if class_name in self.class_name_count:
-            return f'{class_name}_{self.class_name_count[class_name]}'
+            return f"{class_name}_{self.class_name_count[class_name]}"
         return class_name
 
     @staticmethod
     def get_class_name(class_name, class_definition=None):
-        if class_definition and '__class__' in class_definition:
-            return class_definition['__class__']
+        if class_definition and "__class__" in class_definition:
+            return class_definition["__class__"]
         return class_name
 
     def add_field(self, field_name):
         self.field_count += 1
 
-    def get_summary(self, line_separator='\n'):
+    def get_summary(self, line_separator="\n"):
         content = [
-            f'Created {self.class_count} classes',
+            f"Created {self.class_count} classes",
         ]
         if len(self.class_name_count) == self.class_count:
             self.has_duplicate = False
-            content.append(' => No class name duplication found')
+            content.append(" => No class name duplication found")
         else:
             self.has_duplicate = True
-            content.append(f' => We found overlapping class_names ('
-                           f'{self.class_count - len(self.class_name_count)})')
+            content.append(
+                f" => We found overlapping class_names ("
+                f"{self.class_count - len(self.class_name_count)})"
+            )
             for name in self.class_name_count:
                 if self.class_name_count[name] > 1:
-                    content.append(f'   + {name} was defined '
-                                   f'{self.class_name_count[name]} times')
+                    content.append(
+                        f"   + {name} was defined "
+                        f"{self.class_name_count[name]} times"
+                    )
 
-        content.append(f'Defined {self.field_count} fields were found')
+        content.append(f"Defined {self.field_count} fields were found")
         return line_separator.join(content)
 
     def print_summary(self):
@@ -172,11 +187,12 @@ def print_summary(self):
 
 # -----------------------------------------------------------------------------
 
+
 class PythonModule:
-    '''
+    """
     This class generates the Python library of ParFlow keys from the
     yaml files.
-    '''
+    """
 
     SUMMARY_INDEX = 4
 
@@ -191,9 +207,9 @@ def __init__(self, indent=4):
             "'''",
             "from .core import PFDBObj, PFDBObjListNumber",
         ]
-        self.str_indent = ' ' * indent
+        self.str_indent = " " * indent
 
-    def add_line(self, content=''):
+    def add_line(self, content=""):
         self.content.append(content)
 
     def add_separator(self):
@@ -218,157 +234,172 @@ def add_class(self, class_name, class_definition):
 
             validation_summary = self.validation_summary
             dedup_class_name = validation_summary.get_deduplicate_class_name(
-                class_name, class_definition)
-            validation_summary.add_class(validation_summary.get_class_name(
-                class_name, class_definition))
+                class_name, class_definition
+            )
+            validation_summary.add_class(
+                validation_summary.get_class_name(class_name, class_definition)
+            )
 
-            inheritance = 'PFDBObj'
-            if '__inheritance__' in class_definition:
-                inheritance = class_definition['__inheritance__']
+            inheritance = "PFDBObj"
+            if "__inheritance__" in class_definition:
+                inheritance = class_definition["__inheritance__"]
 
-            self.add_line(f'class {dedup_class_name}({inheritance}):')
-            if '__doc__' in class_keys:
-                self.add_comment(class_definition['__doc__'], self.str_indent)
+            self.add_line(f"class {dedup_class_name}({inheritance}):")
+            if "__doc__" in class_keys:
+                self.add_comment(class_definition["__doc__"], self.str_indent)
 
             for key, value in class_definition.items():
                 if is_class(key, class_definition):
                     class_members.append(key)
                 if is_field(key, class_definition):
                     field_members.append(key)
-                if key == '__class_instances__':
-                    class_instances = class_definition['__class_instances__']
+                if key == "__class_instances__":
+                    class_instances = class_definition["__class_instances__"]
                 if is_class_item(key, class_definition):
                     class_items.append(value)
-                    if '__prefix__' in value:
+                    if "__prefix__" in value:
                         field_with_prefix += 1
-                        prefix = value['__prefix__']
+                        prefix = value["__prefix__"]
                         if field_prefix_value and field_prefix_value != prefix:
-                            print('Warning: mismatched prefixes: ',
-                                  f'{field_prefix_value} and {prefix}')
-                            print(f'Using {prefix}...')
+                            print(
+                                "Warning: mismatched prefixes: ",
+                                f"{field_prefix_value} and {prefix}",
+                            )
+                            print(f"Using {prefix}...")
                         field_prefix_value = prefix
                 if is_dynamic(key, class_definition):
-                    class_dynamic[value['__class__']] = value['__from__']
-
-            if (any([class_members, field_members, class_instances,
-                    field_with_prefix, class_dynamic]) or
-                    has_prefix(class_name, class_definition)):
-                '''
-                  def __init__(self, parent=None):
-                    super().__init__(parent)
-                    self.Topology = Topology(self)
-                '''
-                self.add_line(
-                    f'{self.str_indent}def __init__(self, parent=None):')
-                self.add_line(f'{self.str_indent * 2}super().__init__(parent)')
+                    class_dynamic[value["__class__"]] = value["__from__"]
+
+            if any(
+                [
+                    class_members,
+                    field_members,
+                    class_instances,
+                    field_with_prefix,
+                    class_dynamic,
+                ]
+            ) or has_prefix(class_name, class_definition):
+                """
+                def __init__(self, parent=None):
+                  super().__init__(parent)
+                  self.Topology = Topology(self)
+                """
+                self.add_line(f"{self.str_indent}def __init__(self, parent=None):")
+                self.add_line(f"{self.str_indent * 2}super().__init__(parent)")
 
                 if has_value(class_name, class_definition):
-                    self.add_field('_value_', class_definition['__value__'],
-                                   class_details)
+                    self.add_field(
+                        "_value_", class_definition["__value__"], class_details
+                    )
 
                 if has_prefix(class_name, class_definition):
-                    self.add_line(f"{self.str_indent * 2}self._prefix_ = "
-                                  f"'{class_definition['__prefix__']}'")
-                    if inheritance == 'PFDBObjListNumber':
-                        self.add_line(f"{self.str_indent * 2}"
-                                      f"self._details_ = ""{}")
+                    self.add_line(
+                        f"{self.str_indent * 2}self._prefix_ = "
+                        f"'{class_definition['__prefix__']}'"
+                    )
+                    if inheritance == "PFDBObjListNumber":
+                        self.add_line(
+                            f"{self.str_indent * 2}" f"self._details_ = " "{}"
+                        )
 
                 for instance in class_members:
                     name = validation_summary.get_deduplicate_class_name(
-                        instance, class_definition[instance])
-                    self.add_line(f'{self.str_indent * 2}self.{instance} = '
-                                  f'{name}(self)')
+                        instance, class_definition[instance]
+                    )
+                    self.add_line(
+                        f"{self.str_indent * 2}self.{instance} = " f"{name}(self)"
+                    )
 
                 for instance in class_instances:
                     # class_definition[instance]
                     self.add_class_instance(instance)
 
                 for field in field_members:
-                    self.add_field(field, class_definition[field],
-                                   class_details)
+                    self.add_field(field, class_definition[field], class_details)
 
                 if field_with_prefix:
-                    class_details['_prefix_'] = field_prefix_value
+                    class_details["_prefix_"] = field_prefix_value
 
                 self.add_details(class_details)
                 self.add_dynamic(class_dynamic)
 
             for class_member in class_members:
                 # Catch error
-                if class_member == 'help':
-                    print(f'Invalid syntax: {class_name} must use __doc__ '
-                          f'rather than help')
+                if class_member == "help":
+                    print(
+                        f"Invalid syntax: {class_name} must use __doc__ "
+                        f"rather than help"
+                    )
                     sys.exit(1)
                 self.add_class(class_member, class_definition[class_member])
 
             for class_item in class_items:
-                self.add_class(class_item['__class__'], class_item)
+                self.add_class(class_item["__class__"], class_item)
 
         except Exception:
             # traceback.print_exc()
-            print(f'Error when processing class {class_name}')
+            print(f"Error when processing class {class_name}")
 
     def add_details(self, class_details):
         if class_details:
             details_lines = json.dumps(class_details, indent=2).splitlines()
-            line_start = 'self._details_ = '
+            line_start = "self._details_ = "
             for line in details_lines:
-                line_with_indent = f'{self.str_indent * 2}{line_start}{line}'
+                line_with_indent = f"{self.str_indent * 2}{line_start}{line}"
                 self.add_line(json_to_python(line_with_indent))
-                line_start = ''
+                line_start = ""
 
     def add_dynamic(self, dynamic):
         if dynamic:
             dynamic_lines = json.dumps(dynamic, indent=2).splitlines()
-            line_start = 'self._dynamic_ = '
+            line_start = "self._dynamic_ = "
             for line in dynamic_lines:
-                line_with_indent = f'{self.str_indent * 2}{line_start}{line}'
+                line_with_indent = f"{self.str_indent * 2}{line_start}{line}"
                 self.add_line(json_to_python(line_with_indent))
-                line_start = ''
+                line_start = ""
 
-            self.add_line(f'{self.str_indent * 2}self._process_dynamic()')
+            self.add_line(f"{self.str_indent * 2}self._process_dynamic()")
 
     def add_field(self, field_name, field_definition, class_details):
         self.validation_summary.add_field(field_name)
         field_val = None
-        if 'default' in field_definition:
-            field_val = yaml_value(field_definition['default'])
-            field_definition['default'] = field_val
+        if "default" in field_definition:
+            field_val = yaml_value(field_definition["default"])
+            field_definition["default"] = field_val
 
-        self.add_line(f'{self.str_indent * 2}self.{field_name} = '
-                      f'{repr(field_val)}')
+        self.add_line(f"{self.str_indent * 2}self.{field_name} = " f"{repr(field_val)}")
         class_details[field_name] = field_definition
 
     def add_class_instance(self, field_name, instance_definition=None):
-        name = self.validation_summary.get_class_name(field_name,
-                                                      instance_definition)
+        name = self.validation_summary.get_class_name(field_name, instance_definition)
         self.add_line(f"{self.str_indent * 2}self.{field_name} = {name}(self)")
 
     def add_comment(self, doc_content, str_indent):
         self.add_line(f"{str_indent}'''")
         for line in doc_content.splitlines():
-            self.add_line(f'{str_indent}{line}')
+            self.add_line(f"{str_indent}{line}")
         self.add_line(f"{str_indent}'''")
 
     def add_dict(self, name, d):
         # Adds a dict at file scope
         self.add_separator()
         json_data = json.dumps(d, indent=2)
-        line_start = f'{name} = '
+        line_start = f"{name} = "
         for line in json_data.splitlines():
-            self.add_line(json_to_python(f'{line_start}{line}'))
-            line_start = ''
+            self.add_line(json_to_python(f"{line_start}{line}"))
+            line_start = ""
 
-    def get_content(self, line_separator='\n'):
-        self.content[self.SUMMARY_INDEX] = (
-            self.validation_summary.get_summary(line_separator))
+    def get_content(self, line_separator="\n"):
+        self.content[self.SUMMARY_INDEX] = self.validation_summary.get_summary(
+            line_separator
+        )
         # Ensure new line at the end
         if self.content[-1]:
-            self.content.append('')
+            self.content.append("")
 
         return line_separator.join(self.content)
 
-    def write(self, file_path, line_separator='\n'):
+    def write(self, file_path, line_separator="\n"):
         content = self.get_content(line_separator)
         Path(file_path).write_text(content)
 
@@ -377,6 +408,7 @@ def write(self, file_path, line_separator='\n'):
 # API to generate library module
 # -----------------------------------------------------------------------------
 
+
 def generate_module_from_definitions(definitions):
     generated_module = PythonModule()
 
@@ -394,10 +426,12 @@ def generate_module_from_definitions(definitions):
 # API to generate CLM key translation dictionary
 # -----------------------------------------------------------------------------
 
+
 def find_paths_to_key(d, search_key):
     """Recursively search a dict for a particular key
-       This returns a path to each key found
+    This returns a path to each key found
     """
+
     def _recursive_find(cur, cur_path, paths):
         if isinstance(cur, dict):
             for key, val in cur.items():
@@ -413,9 +447,9 @@ def _recursive_find(cur, cur_path, paths):
 
 # -----------------------------------------------------------------------------
 
+
 def recursive_get(d, path):
-    """Get a value in a dictionary from a path
-    """
+    """Get a value in a dictionary from a path"""
     for entry in path:
         d = d[entry]
 
@@ -424,13 +458,14 @@ def recursive_get(d, path):
 
 # -----------------------------------------------------------------------------
 
+
 def generate_clm_key_dict(source_file):
-    with open(source_file, 'r') as rf:
+    with open(source_file, "r") as rf:
         data = yaml.safe_load(rf)
 
-    paths = find_paths_to_key(data, 'clm_key')
+    paths = find_paths_to_key(data, "clm_key")
 
-    return {recursive_get(data, x + ['clm_key']): x for x in paths}
+    return {recursive_get(data, x + ["clm_key"]): x for x in paths}
 
 
 # -----------------------------------------------------------------------------
@@ -439,21 +474,20 @@ def generate_clm_key_dict(source_file):
 
 if __name__ == "__main__":
     core_definitions = YAML_MODULES_TO_PROCESS
-    def_path = Path(__file__).resolve().parent.parent / 'definitions'
-    definition_files = [
-        def_path / f'{module}.yaml' for module in core_definitions]
+    def_path = Path(__file__).resolve().parent.parent / "definitions"
+    definition_files = [def_path / f"{module}.yaml" for module in core_definitions]
     output_file_path = Path(sys.argv[1]).resolve()
-    clm_key_file_name = Path(def_path) / 'solver.yaml'
+    clm_key_file_name = Path(def_path) / "solver.yaml"
 
-    print('-' * 80)
-    print('Generate Parflow database module')
-    print('-' * 80)
+    print("-" * 80)
+    print("Generate Parflow database module")
+    print("-" * 80)
     generated_module = generate_module_from_definitions(definition_files)
     print(generated_module.validation_summary.get_summary())
     # Write out the clm dict as well
     clm_key_dict = generate_clm_key_dict(clm_key_file_name)
-    generated_module.add_dict('CLM_KEY_DICT', clm_key_dict)
-    print('-' * 80)
+    generated_module.add_dict("CLM_KEY_DICT", clm_key_dict)
+    print("-" * 80)
     generated_module.write(output_file_path)
 
     if generated_module.validation_summary.has_duplicate:
diff --git a/pf-keys/generators/read_the_doc_rst.py b/pf-keys/generators/read_the_doc_rst.py
index 87b32d015..f8b322590 100644
--- a/pf-keys/generators/read_the_doc_rst.py
+++ b/pf-keys/generators/read_the_doc_rst.py
@@ -1,7 +1,7 @@
-r'''
+r"""
 This module provide the infrastructure to load and generate the Parflow
 database structure as documentation files for Read The Docs.
-'''
+"""
 
 from pathlib import Path
 import yaml
@@ -9,23 +9,23 @@
 # -----------------------------------------------------------------------------
 
 YAML_MODULES_TO_PROCESS = [
-    'core',
-    'geom',
-    'solver',
-    'wells',
-    'phase',
-    'timing',
-    'netcdf',
-    'bconditions',
-    'run'
+    "core",
+    "geom",
+    "solver",
+    "wells",
+    "phase",
+    "timing",
+    "netcdf",
+    "bconditions",
+    "run",
 ]
 
 # -----------------------------------------------------------------------------
 
 LEVELS = [
-    '=',
-    '-',
-    '^',
+    "=",
+    "-",
+    "^",
     '"',
     '"',
     '"',
@@ -35,109 +35,123 @@
 
 # -----------------------------------------------------------------------------
 
+
 def handle_domain(name, definition):
-    '''
+    """
     This method will extract information from a domain and present it
     for the documentation.
-    '''
-    indent_str = ' ' * 4
+    """
+    indent_str = " " * 4
     lines = []
     list_count = 0
 
-    if name == 'MandatoryValue':
-        lines.append(f'{indent_str}The value is required')
+    if name == "MandatoryValue":
+        lines.append(f"{indent_str}The value is required")
 
-    if name == 'IntValue':
-        lines.append(f'{indent_str}The value must be an Integer')
-        if definition and 'min_value' in definition:
+    if name == "IntValue":
+        lines.append(f"{indent_str}The value must be an Integer")
+        if definition and "min_value" in definition:
             list_count += 1
-            lines.append(f'{indent_str}  - with a value greater than or equal '
-                         f'to {definition["min_value"]}')
-        if definition and 'max_value' in definition:
+            lines.append(
+                f"{indent_str}  - with a value greater than or equal "
+                f'to {definition["min_value"]}'
+            )
+        if definition and "max_value" in definition:
             list_count += 1
-            lines.append(f'{indent_str}  - with a value less than or equal to '
-                         f'{definition["max_value"]}')
-
-    if name == 'DoubleValue':
-        lines.append(f'{indent_str}The value must be a Double')
-        if definition and 'min_value' in definition:
+            lines.append(
+                f"{indent_str}  - with a value less than or equal to "
+                f'{definition["max_value"]}'
+            )
+
+    if name == "DoubleValue":
+        lines.append(f"{indent_str}The value must be a Double")
+        if definition and "min_value" in definition:
             list_count += 1
-            lines.append(f'{indent_str}  - with a value greater than or equal '
-                         f'to {definition["min_value"]}')
-        if definition and 'max_value' in definition:
+            lines.append(
+                f"{indent_str}  - with a value greater than or equal "
+                f'to {definition["min_value"]}'
+            )
+        if definition and "max_value" in definition:
             list_count += 1
-            lines.append(f'{indent_str}  - with a value less than or equal to '
-                         f'{definition["max_value"]}')
-        if definition and 'neg_int' in definition:
+            lines.append(
+                f"{indent_str}  - with a value less than or equal to "
+                f'{definition["max_value"]}'
+            )
+        if definition and "neg_int" in definition:
             list_count += 1
-            lines.append(f'{indent_str}  - must be an integer if less than 0')
+            lines.append(f"{indent_str}  - must be an integer if less than 0")
 
-    if name == 'EnumDomain':
-        lines.append(f'{indent_str}The value must be one of the following '
-                     f'options: {(", ".join(definition["enum_list"]))}')
+    if name == "EnumDomain":
+        lines.append(
+            f"{indent_str}The value must be one of the following "
+            f'options: {(", ".join(definition["enum_list"]))}'
+        )
 
-    if name == 'AnyString':
-        lines.append(f'{indent_str}The value must be a string')
+    if name == "AnyString":
+        lines.append(f"{indent_str}The value must be a string")
 
-    if name == 'BoolDomain':
-        lines.append(f'{indent_str}The value must be True or False')
+    if name == "BoolDomain":
+        lines.append(f"{indent_str}The value must be True or False")
 
-    if name == 'RequiresModule':
-        lines.append(f'{indent_str}This key requires the availability of the '
-                     f'following module(s) in ParFlow: {definition}')
+    if name == "RequiresModule":
+        lines.append(
+            f"{indent_str}This key requires the availability of the "
+            f"following module(s) in ParFlow: {definition}"
+        )
 
-    if name == 'Deprecated':
-        lines.append('')
-        lines.append('.. warning::')
-        lines.append(f'    This key will be deprecated in v{definition}')
+    if name == "Deprecated":
+        lines.append("")
+        lines.append(".. warning::")
+        lines.append(f"    This key will be deprecated in v{definition}")
 
-    if name == 'Removed':
-        lines.append('')
-        lines.append('.. warning::')
-        lines.append(f'    This key will be removed in v{definition}')
+    if name == "Removed":
+        lines.append("")
+        lines.append(".. warning::")
+        lines.append(f"    This key will be removed in v{definition}")
 
     if list_count:
-        lines.append('')
+        lines.append("")
 
-    return '\n'.join(lines)
+    return "\n".join(lines)
 
 
 # -----------------------------------------------------------------------------
 
+
 class RST_module:
-    '''
+    """
     Helper class that can be used to create a RST file for ReadTheDoc
-    '''
+    """
+
     def __init__(self, title):
         self.content = [
-            '*' * 80,
+            "*" * 80,
             title,
-            '*' * 80,
+            "*" * 80,
         ]
 
-    def add_line(self, content=''):
+    def add_line(self, content=""):
         self.content.append(content)
 
     def add_section(self, level, prefix, key, sub_section):
-        if prefix and prefix != 'BaseRun':
-            title = f'{prefix}.{key}'
+        if prefix and prefix != "BaseRun":
+            title = f"{prefix}.{key}"
         else:
             title = key
 
-        if key == '__value__':
+        if key == "__value__":
             title = prefix
 
-        warning = ''
-        if '__rst__' in sub_section:
-            if 'name' in sub_section['__rst__']:
-                title = sub_section['__rst__']['name']
-            if 'warning' in sub_section['__rst__']:
-                warning = sub_section['__rst__']['warning']
-            if 'skip' in sub_section['__rst__']:
+        warning = ""
+        if "__rst__" in sub_section:
+            if "name" in sub_section["__rst__"]:
+                title = sub_section["__rst__"]["name"]
+            if "warning" in sub_section["__rst__"]:
+                warning = sub_section["__rst__"]["warning"]
+            if "skip" in sub_section["__rst__"]:
                 for sub_key in sub_section:
-                    if not sub_key.startswith('_') or sub_key == '__value__':
-                        self.add_section(level, title, sub_key,
-                                         sub_section[sub_key])
+                    if not sub_key.startswith("_") or sub_key == "__value__":
+                        self.add_section(level, title, sub_key, sub_section[sub_key])
                 return
 
         self.add_line()
@@ -145,47 +159,47 @@ def add_section(self, level, prefix, key, sub_section):
         self.add_line(LEVELS[level] * 80)
         self.add_line()
         if warning:
-            self.add_line('.. warning::')
-            self.add_line(f'    {warning}')
+            self.add_line(".. warning::")
+            self.add_line(f"    {warning}")
 
         leaf = False
-        description = ''
+        description = ""
 
-        if 'help' in sub_section:
+        if "help" in sub_section:
             leaf = True
-            description = sub_section['help']
+            description = sub_section["help"]
 
-        if '__doc__' in sub_section:
-            description = sub_section['__doc__']
+        if "__doc__" in sub_section:
+            description = sub_section["__doc__"]
 
         self.add_line(description)
         self.add_line()
 
         if leaf:
             # Need to process domains and more...
-            if 'default' in sub_section:
+            if "default" in sub_section:
                 self.add_line(f':default: {sub_section["default"]}')
 
-            if 'domains' in sub_section:
-                self.add_line('.. note::')
-                for domain, val in sub_section['domains'].items():
+            if "domains" in sub_section:
+                self.add_line(".. note::")
+                for domain, val in sub_section["domains"].items():
                     self.add_line(handle_domain(domain, val))
                 self.add_line()
 
         else:
             # Keep adding sections
             for sub_key, val in sub_section.items():
-                if not sub_key.startswith('_') or sub_key == '__value__':
+                if not sub_key.startswith("_") or sub_key == "__value__":
                     self.add_section(level + 1, title, sub_key, val)
 
-    def get_content(self,  line_separator='\n'):
+    def get_content(self, line_separator="\n"):
         # Ensure new line at the end
         if self.content[-1]:
-            self.content.append('')
+            self.content.append("")
 
         return line_separator.join(self.content)
 
-    def write(self, file_path, line_separator='\n'):
+    def write(self, file_path, line_separator="\n"):
         content = self.get_content(line_separator)
         Path(file_path).write_text(content)
 
@@ -194,15 +208,16 @@ def write(self, file_path, line_separator='\n'):
 # Expected API to use
 # -----------------------------------------------------------------------------
 
+
 def generate_module_from_definitions(definitions):
-    generated_RST = RST_module('ParFlow Key Documentation')
+    generated_RST = RST_module("ParFlow Key Documentation")
 
     for yaml_file in definitions:
         with open(yaml_file) as file:
             yaml_struct = yaml.safe_load(file)
 
         for key, val in yaml_struct.items():
-            generated_RST.add_section(0, '', key, val)
+            generated_RST.add_section(0, "", key, val)
 
     return generated_RST
 
@@ -214,15 +229,13 @@ def generate_module_from_definitions(definitions):
 if __name__ == "__main__":
     core_definitions = YAML_MODULES_TO_PROCESS
     base_path = Path(__file__).resolve().parent
-    def_path = base_path.parent / 'definitions'
-    definition_files = [
-        def_path / f'{module}.yaml' for module in core_definitions]
-    output_file_path = (
-        base_path.parent.parent / 'docs/pf-keys/parflow/keys.rst')
-
-    print('-' * 80)
-    print('Generate ParFlow database documentation')
-    print('-' * 80)
+    def_path = base_path.parent / "definitions"
+    definition_files = [def_path / f"{module}.yaml" for module in core_definitions]
+    output_file_path = base_path.parent.parent / "docs/pf-keys/parflow/keys.rst"
+
+    print("-" * 80)
+    print("Generate ParFlow database documentation")
+    print("-" * 80)
     generated_module = generate_module_from_definitions(definition_files)
-    print('-' * 80)
+    print("-" * 80)
     generated_module.write(output_file_path)
diff --git a/pf-keys/generators/simput/requirements.txt b/pf-keys/generators/simput/requirements.txt
index 55a57b3de..9fb5f3765 100644
--- a/pf-keys/generators/simput/requirements.txt
+++ b/pf-keys/generators/simput/requirements.txt
@@ -2,4 +2,4 @@ click==8.0.1
 importlib-metadata==4.4.0
 PyYAML==6.0.1
 typing-extensions==3.10.0.0
-zipp==3.4.1
+zipp==3.19.1
diff --git a/pf-keys/generators/simput/simput_model.py b/pf-keys/generators/simput/simput_model.py
index f18eca920..e3a6a4b52 100644
--- a/pf-keys/generators/simput/simput_model.py
+++ b/pf-keys/generators/simput/simput_model.py
@@ -45,6 +45,7 @@
     "definitions": {},
 }
 
+
 # Name Parameter
 def name_param(att_name):
     parent, param_id = att_name.split("/")
diff --git a/pfsimulator/amps/common/amps_abort.c b/pfsimulator/amps/common/amps_abort.c
index 39f4f4fa8..abe93447c 100644
--- a/pfsimulator/amps/common/amps_abort.c
+++ b/pfsimulator/amps/common/amps_abort.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <amps.h>
 
diff --git a/pfsimulator/amps/common/amps_clear.c b/pfsimulator/amps/common/amps_clear.c
index b5d97b033..fc96f5c49 100644
--- a/pfsimulator/amps/common/amps_clear.c
+++ b/pfsimulator/amps/common/amps_clear.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <string.h>
 #include <stdarg.h>
 
diff --git a/pfsimulator/amps/common/amps_clock.c b/pfsimulator/amps/common/amps_clock.c
index 0dcc4d78a..04d52c65d 100644
--- a/pfsimulator/amps/common/amps_clock.c
+++ b/pfsimulator/amps/common/amps_clock.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/common/amps_common.h b/pfsimulator/amps/common/amps_common.h
index 6a750060f..22f3162d5 100644
--- a/pfsimulator/amps/common/amps_common.h
+++ b/pfsimulator/amps/common/amps_common.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #ifndef amps_common_include
 #define amps_common_include
 
diff --git a/pfsimulator/amps/common/amps_exchange.c b/pfsimulator/amps/common/amps_exchange.c
index 563793a8f..9c1635710 100644
--- a/pfsimulator/amps/common/amps_exchange.c
+++ b/pfsimulator/amps/common/amps_exchange.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 
 #include "amps.h"
diff --git a/pfsimulator/amps/common/amps_ffopen.c b/pfsimulator/amps/common/amps_ffopen.c
index ed243e597..97158139a 100644
--- a/pfsimulator/amps/common/amps_ffopen.c
+++ b/pfsimulator/amps/common/amps_ffopen.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <unistd.h>
 #include <string.h>
@@ -205,20 +205,20 @@ amps_File amps_FFopen(amps_Comm comm, char *filename, char *type, long size)
       AMPS_ABORT("AMPS Error");
     }
 
-    if(fscanf(file, "%ld", &start) != 1)
+    if (fscanf(file, "%ld", &start) != 1)
     {
       printf("AMPS Error: Can't read start in file %s\n", dist_filename);
       AMPS_ABORT("AMPS Error");
     }
-    
+
     for (p = 1; p < amps_Size(comm); p++)
     {
-      if(fscanf(file, "%ld", &start) != 1)
+      if (fscanf(file, "%ld", &start) != 1)
       {
-	printf("AMPS Error: Can't read start in file %s\n", dist_filename);
-	AMPS_ABORT("AMPS Error");
+        printf("AMPS Error: Can't read start in file %s\n", dist_filename);
+        AMPS_ABORT("AMPS Error");
       }
-      
+
       amps_Send(comm, p, invoice);
     }
     fclose(file);
diff --git a/pfsimulator/amps/common/amps_find_powers.c b/pfsimulator/amps/common/amps_find_powers.c
index 5edeea177..04b6beae2 100644
--- a/pfsimulator/amps/common/amps_find_powers.c
+++ b/pfsimulator/amps/common/amps_find_powers.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/common/amps_fopen.c b/pfsimulator/amps/common/amps_fopen.c
index f1a39d60d..35d023aad 100644
--- a/pfsimulator/amps/common/amps_fopen.c
+++ b/pfsimulator/amps/common/amps_fopen.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/common/amps_invoice.c b/pfsimulator/amps/common/amps_invoice.c
index a9b8a6271..f4b8c4ed4 100644
--- a/pfsimulator/amps/common/amps_invoice.c
+++ b/pfsimulator/amps/common/amps_invoice.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <string.h>
 #include <ctype.h>
diff --git a/pfsimulator/amps/common/amps_io.c b/pfsimulator/amps/common/amps_io.c
index aec36bfee..a64008554 100644
--- a/pfsimulator/amps/common/amps_io.c
+++ b/pfsimulator/amps/common/amps_io.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -40,7 +40,7 @@ void amps_ScanByte(
   for (end_ptr = data + len * stride, ptr = data; ptr < end_ptr;
        ptr += stride)
   {
-    if(fread(ptr, 1, 1, file) != 1)
+    if (fread(ptr, 1, 1, file) != 1)
     {
       printf("AMPS Error: Can't read byte\n");
       AMPS_ABORT("AMPS Error");
@@ -61,7 +61,7 @@ void amps_ScanChar(
   for (end_ptr = data + len * stride, ptr = data; ptr < end_ptr;
        ptr += stride)
   {
-    if(fscanf(file, "%c", ptr) != 1)
+    if (fscanf(file, "%c", ptr) != 1)
     {
       printf("AMPS Error: Can't read char\n");
       AMPS_ABORT("AMPS Error");
@@ -81,7 +81,7 @@ void amps_ScanShort(
   for (end_ptr = data + len * stride, ptr = data; ptr < end_ptr;
        ptr += stride)
   {
-    if(fscanf(file, "%hd ", ptr) != 1)
+    if (fscanf(file, "%hd ", ptr) != 1)
     {
       printf("AMPS Error: Can't read short\n");
       AMPS_ABORT("AMPS Error");
@@ -101,7 +101,7 @@ void amps_ScanInt(
   for (end_ptr = data + len * stride, ptr = data; ptr < end_ptr;
        ptr += stride)
   {
-    if(fscanf(file, "%d ", ptr) != 1)
+    if (fscanf(file, "%d ", ptr) != 1)
     {
       printf("AMPS Error: Can't read int\n");
       AMPS_ABORT("AMPS Error");
@@ -121,7 +121,7 @@ void amps_ScanLong(
   for (end_ptr = data + len * stride, ptr = data; ptr < end_ptr;
        ptr += stride)
   {
-    if(fscanf(file, "%ld ", ptr) != 1)
+    if (fscanf(file, "%ld ", ptr) != 1)
     {
       printf("AMPS Error: Can't read long\n");
       AMPS_ABORT("AMPS Error");
@@ -141,12 +141,11 @@ void amps_ScanFloat(
   for (end_ptr = data + len * stride, ptr = data; ptr < end_ptr;
        ptr += stride)
   {
-    if(fscanf(file, "%f ", ptr) != 1)
+    if (fscanf(file, "%f ", ptr) != 1)
     {
       printf("AMPS Error: Can't read float\n");
       AMPS_ABORT("AMPS Error");
     }
-
   }
 }
 
@@ -162,7 +161,7 @@ void amps_ScanDouble(
   for (end_ptr = data + len * stride, ptr = data; ptr < end_ptr;
        ptr += stride)
   {
-    if(fscanf(file, "%lf ", ptr) != 1)
+    if (fscanf(file, "%lf ", ptr) != 1)
     {
       printf("AMPS Error: Can't read double\n");
       AMPS_ABORT("AMPS Error");
@@ -241,7 +240,7 @@ void amps_ReadDouble(amps_File file, double *ptr, int len)
   /* read in each double with bytes swaped                               */
   for (i = len, data = ptr; i--;)
   {
-    if(fread(&a.number, sizeof(double), 1, (FILE*)file) != 1)
+    if (fread(&a.number, sizeof(double), 1, (FILE*)file) != 1)
     {
       printf("AMPS Error: Can't read double\n");
       AMPS_ABORT("AMPS Error");
@@ -274,7 +273,7 @@ void amps_ReadInt(amps_File file, int *ptr, int len)
 
   for (i = len, data = ptr; i--;)
   {
-    if(fread(&a.number, sizeof(int), 1, (FILE*)file) != 1)
+    if (fread(&a.number, sizeof(int), 1, (FILE*)file) != 1)
     {
       printf("AMPS Error: Can't read int\n");
       AMPS_ABORT("AMPS Error");
@@ -322,12 +321,12 @@ int len;
 
   for (i = len, data = ptr; i--;)
   {
-    if(fread(&number, sizeof(short), 1, (FILE*)file) != 1)
+    if (fread(&number, sizeof(short), 1, (FILE*)file) != 1)
     {
       printf("AMPS Error: Can't read byte\n");
       AMPS_ABORT("AMPS Error");
     }
-    
+
     *data++ = number;
   }
 }
diff --git a/pfsimulator/amps/common/amps_newhandle.c b/pfsimulator/amps/common/amps_newhandle.c
index da80fda89..d2ddecb5f 100644
--- a/pfsimulator/amps/common/amps_newhandle.c
+++ b/pfsimulator/amps/common/amps_newhandle.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/common/amps_newpackage.c b/pfsimulator/amps/common/amps_newpackage.c
index e628ce48f..659f7c85e 100644
--- a/pfsimulator/amps/common/amps_newpackage.c
+++ b/pfsimulator/amps/common/amps_newpackage.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/common/amps_print.c b/pfsimulator/amps/common/amps_print.c
index 6cf29d77d..92cbf8e64 100644
--- a/pfsimulator/amps/common/amps_print.c
+++ b/pfsimulator/amps/common/amps_print.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <stdarg.h>
 
 #include "amps.h"
@@ -34,28 +34,29 @@ FILE* amps_console = NULL;
 /**
  * Set the stream for ParFlow output messages.
  *
- * By default the stream is NULL.  Messages 
+ * By default the stream is NULL.  Messages
  * will be lost if the console is not set.
  *
- * \param stream 
+ * \param stream
  * \return The previous stream
  */
 FILE* amps_SetConsole(FILE* stream)
 {
   FILE* prev_stream = amps_console;
+
   amps_console = stream;
   return prev_stream;
 }
 
 /**
- * This routine is used to print information and error messages 
+ * This routine is used to print information and error messages
  * generated by ParFlow.
  *
- * The destination is controlled by setting a file stream (FILE*) 
+ * The destination is controlled by setting a file stream (FILE*)
  * to be used.  When ParFlow is used in an embedded setting (being called
- * as a library inside another application) this may be used 
+ * as a library inside another application) this may be used
  * to direct ParFlow messages.
- * 
+ *
  * The arguments are the same as for the standard C {\bf printf} function.
  *
  * {\large Example:}
diff --git a/pfsimulator/amps/common/amps_sfbcast.c b/pfsimulator/amps/common/amps_sfbcast.c
index 1a7a72290..0ddf0a8b8 100644
--- a/pfsimulator/amps/common/amps_sfbcast.c
+++ b/pfsimulator/amps/common/amps_sfbcast.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -94,7 +94,7 @@ int amps_SFBCast(amps_Comm comm, amps_File file, amps_Invoice invoice)
 
       switch (ptr->type)
       {
-	case AMPS_INVOICE_BYTE_CTYPE:
+        case AMPS_INVOICE_BYTE_CTYPE:
           if (ptr->data_type == AMPS_INVOICE_POINTER)
           {
             *((void**)(ptr->data)) = (void*)malloc(sizeof(char) * (size_t)(len * stride));
diff --git a/pfsimulator/amps/common/amps_sfclose.c b/pfsimulator/amps/common/amps_sfclose.c
index 8153fe261..3359659f2 100644
--- a/pfsimulator/amps/common/amps_sfclose.c
+++ b/pfsimulator/amps/common/amps_sfclose.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/common/amps_sfopen.c b/pfsimulator/amps/common/amps_sfopen.c
index a489bbd69..a4253b5a8 100644
--- a/pfsimulator/amps/common/amps_sfopen.c
+++ b/pfsimulator/amps/common/amps_sfopen.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/common/amps_wait.c b/pfsimulator/amps/common/amps_wait.c
index e641790b9..8e7dd9b15 100644
--- a/pfsimulator/amps/common/amps_wait.c
+++ b/pfsimulator/amps/common/amps_wait.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/common/signal.c b/pfsimulator/amps/common/signal.c
index f61603925..5216d67cc 100644
--- a/pfsimulator/amps/common/signal.c
+++ b/pfsimulator/amps/common/signal.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <signal.h>
 #include <math.h>
@@ -143,10 +143,10 @@ void Fsignal()
 #ifdef AMPS_IRIX_SIGNALS
   signal(SIGFPE, handler_fpe);      /* Arithmetic exception        */
   handle_sigfpes(_ON, _EN_DIVZERO, 0, _REPLACE_HANDLER_ON_ERROR,
-                 (void (*)())handler_division);
+                 (void (*)()) handler_division);
   handle_sigfpes(_ON, _EN_OVERFL, 0, _REPLACE_HANDLER_ON_ERROR,
-                 (void (*)())handler_overflow);
+                 (void (*)()) handler_overflow);
   handle_sigfpes(_ON, _EN_INVALID, 0, _REPLACE_HANDLER_ON_ERROR,
-                 (void (*)())handler_invalid);
+                 (void (*)()) handler_invalid);
 #endif
 }
diff --git a/pfsimulator/amps/cuda/amps.h b/pfsimulator/amps/cuda/amps.h
index 771c30d54..23e15996e 100644
--- a/pfsimulator/amps/cuda/amps.h
+++ b/pfsimulator/amps/cuda/amps.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #ifndef amps_include
 #define amps_include
 
@@ -271,7 +271,6 @@ extern int amps_size;
  *--------------------------------------------------------------------------*/
 #define amps_device_max_streams 10
 typedef struct amps_devicestruct {
-
   char *combuf_recv;
   char *combuf_send;
   long combuf_recv_size;
@@ -279,7 +278,6 @@ typedef struct amps_devicestruct {
 
   int streams_created;
   cudaStream_t stream[amps_device_max_streams];
-
 } amps_Devicestruct;
 
 extern amps_Devicestruct amps_device_globals;
@@ -397,219 +395,219 @@ extern amps_Buffer *amps_BufferFreeList;
 /*---------------------------------------------------------------------------*/
 /* Functions to for align                                                    */
 /*---------------------------------------------------------------------------*/
-#define AMPS_ALIGN(type, src, dest, len, stride) \
-  ((sizeof(type) -                               \
-    ((unsigned long)(dest) % sizeof(type)))      \
-   % sizeof(type));
+#define AMPS_ALIGN(type, src, dest, len, stride)       \
+        ((sizeof(type) -                               \
+          ((unsigned long)(dest) % sizeof(type)))      \
+         % sizeof(type));
 
 
 #define AMPS_CALL_BYTE_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(char, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(char, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_CHAR_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(char, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(char, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_SHORT_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(short, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(short, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_INT_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(int, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(int, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_LONG_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(long, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(long, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_FLOAT_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(float, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(float, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_DOUBLE_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(double, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(double, (_src), (_dest), (_len), (_stride))
 
 /*---------------------------------------------------------------------------*/
 /* Functions to for sizeof                                                   */
 /*---------------------------------------------------------------------------*/
 #define AMPS_SIZEOF(len, stride, size) \
-  (size_t)(len) * (size)
+        (size_t)(len) * (size)
 
-#define AMPS_CALL_BYTE_SIZEOF(_comm, _src, _dest, _len, _stride)	\
-  AMPS_SIZEOF((_len), (_stride), sizeof(char))
+#define AMPS_CALL_BYTE_SIZEOF(_comm, _src, _dest, _len, _stride)        \
+        AMPS_SIZEOF((_len), (_stride), sizeof(char))
 
 #define AMPS_CALL_CHAR_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(char))
+        AMPS_SIZEOF((_len), (_stride), sizeof(char))
 
 #define AMPS_CALL_SHORT_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(short))
+        AMPS_SIZEOF((_len), (_stride), sizeof(short))
 
 #define AMPS_CALL_INT_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(int))
+        AMPS_SIZEOF((_len), (_stride), sizeof(int))
 
 #define AMPS_CALL_LONG_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(long))
+        AMPS_SIZEOF((_len), (_stride), sizeof(long))
 
 #define AMPS_CALL_FLOAT_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(float))
+        AMPS_SIZEOF((_len), (_stride), sizeof(float))
 
 #define AMPS_CALL_DOUBLE_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(double))
+        AMPS_SIZEOF((_len), (_stride), sizeof(double))
 
 /*---------------------------------------------------------------------------*/
 /*---------------------------------------------------------------------------*/
 
 #ifdef __CUDACC__
-#define AMPS_CONVERT_OUT(type_, cvt, comm, src, dest, len, stride)                                           \
-{                                                                                                            \
-  type_ *ptr_src, *ptr_dest;                                                                                 \
-  cudaPointerAttributes attributes;                                                                          \
-  cudaPointerGetAttributes(&attributes, src);                                                                \
-  if(cudaGetLastError() != cudaSuccess || attributes.type < 2){                                              \
-    if ((char*)(src) != (char*)(dest))                                                                       \
-      if ((stride) == 1)                                                                                     \
-        bcopy((src), (dest), (len) * sizeof(type_));                                                         \
-      else                                                                                                   \
-        for (ptr_src = (type_*)(src), ptr_dest = (type_*)(dest); ptr_src < (type_*)(src) + (len) * (stride); \
-            ptr_src += (stride), ptr_dest++)                                                                 \
-          bcopy((ptr_src), (ptr_dest), sizeof(type_));                                                       \
-  }else{                                                                                                     \
-    if ((char*)(src) != (char*)(dest))                                                                       \
-    {                                                                                                        \
-      const int blocksize = 1024;                                                                            \
-      StridedCopyKernel<<<(len + blocksize - 1)/blocksize, blocksize>>>(                                     \
-        (type_*)(dest), 1, (type_*)(src), stride, len);                                                      \
-      CUDA_ERRCHK(cudaPeekAtLastError());                                                                       \
-      CUDA_ERRCHK(cudaStreamSynchronize(0));                                                                    \
-    }                                                                                                        \
-  }                                                                                                          \
-}
-
-#define AMPS_CONVERT_IN(type_, cvt, comm, src, dest, len, stride)                                            \
-{                                                                                                            \
-  char *ptr_src, *ptr_dest;                                                                                  \
-  cudaPointerAttributes attributes;                                                                          \
-  cudaPointerGetAttributes(&attributes, src);                                                                \
-  if(cudaGetLastError() != cudaSuccess || attributes.type < 2){                                              \
-  if ((src) != (dest))                                                                                       \
-  {                                                                                                          \
-    if ((stride) == 1)                                                                                       \
-    {                                                                                                        \
-      bcopy((src), (dest), (size_t)(len) * sizeof(type_));                                                   \
-    }                                                                                                        \
-    else                                                                                                     \
-    {                                                                                                        \
-      for (ptr_src = (char*)(src), (ptr_dest) = (char*)(dest);                                               \
-            (ptr_dest) < (char*)(dest) + (size_t)((len) * (stride)) * sizeof(type_);                         \
-            (ptr_src) += sizeof(type_), (ptr_dest) += sizeof(type_) * (size_t)((stride)))                    \
-        bcopy(ptr_src, ptr_dest, sizeof(type_));                                                             \
-    }                                                                                                        \
-  }                                                                                                          \
-  }else{                                                                                                     \
-    if ((src) != (dest))                                                                                     \
-    {                                                                                                        \
-      const int blocksize = 1024;                                                                            \
-      StridedCopyKernel<<<(len + blocksize - 1)/blocksize, blocksize>>>(                                     \
-        (type_*)(dest), stride, (type_*)(src), 1, len);                                                      \
-      CUDA_ERRCHK(cudaPeekAtLastError());                                                                       \
-      CUDA_ERRCHK(cudaStreamSynchronize(0));                                                                    \
-    }                                                                                                        \
-  }                                                                                                          \
-}  
-#else   
-#define AMPS_CONVERT_OUT(type, cvt, comm, src, dest, len, stride)                                            \
-{                                                                                                            \
-  type *ptr_src, *ptr_dest;                                                                                  \
-  if ((char*)(src) != (char*)(dest))                                                                         \
-    if ((stride) == 1)                                                                                       \
-      bcopy((src), (dest), (len) * sizeof(type));                                                            \
-    else                                                                                                     \
-      for (ptr_src = (type*)(src), ptr_dest = (type*)(dest); ptr_src < (type*)(src) + (len) * (stride);      \
-            ptr_src += (stride), ptr_dest++)                                                                 \
-        bcopy((ptr_src), (ptr_dest), sizeof(type));                                                          \
-}
-
-#define AMPS_CONVERT_IN(type, cvt, comm, src, dest, len, stride)                                             \
-{                                                                                                            \
-  char *ptr_src, *ptr_dest;                                                                                  \
-  if ((src) != (dest))                                                                                       \
-  {                                                                                                          \
-    if ((stride) == 1)                                                                                       \
-    {                                                                                                        \
-      bcopy((src), (dest), (size_t)(len) * sizeof(type));                                                    \
-    }                                                                                                        \
-    else                                                                                                     \
-    {                                                                                                        \
-      for (ptr_src = (char*)(src), (ptr_dest) = (char*)(dest);                                               \
-            (ptr_dest) < (char*)(dest) + (size_t)((len) * (stride)) * sizeof(type);                          \
-            (ptr_src) += sizeof(type), (ptr_dest) += sizeof(type) * (size_t)((stride)))                      \
-        bcopy(ptr_src, ptr_dest, sizeof(type));                                                              \
-      ;                                                                                                      \
-    }                                                                                                        \
-  }                                                                                                          \
-}
+#define AMPS_CONVERT_OUT(type_, cvt, comm, src, dest, len, stride)                                                                                                                           \
+        {                                                                                                                                                                                    \
+          type_ *ptr_src, *ptr_dest;                                                                                                                                                         \
+          cudaPointerAttributes attributes;                                                                                                                                                  \
+          cudaPointerGetAttributes(&attributes, src);                                                                                                                                        \
+          if (cudaGetLastError() != cudaSuccess || attributes.type < 2) {                                                                                                                    \
+            if ((char*)(src) != (char*)(dest))                                                                                                                                               \
+            if ((stride) == 1)                                                                                                                                                               \
+            bcopy((src), (dest), (len) * sizeof(type_));                                                                                                                                     \
+            else                                                                                                                                                                             \
+            for (ptr_src = (type_*)(src), ptr_dest = (type_*)(dest); ptr_src < (type_*)(src) + (len) * (stride);                                                                             \
+                 ptr_src += (stride), ptr_dest++)                                                                                                                                            \
+            bcopy((ptr_src), (ptr_dest), sizeof(type_));                                                                                                                                     \
+          }else{                                                                                                                                                                             \
+            if ((char*)(src) != (char*)(dest))                                                                                                                                               \
+            {                                                                                                                                                                                \
+              const int blocksize = 1024;                                                                                                                                                    \
+              StridedCopyKernel << < (len + blocksize - 1) / blocksize, blocksize >> > (                                                                                                     \
+                                                                                        (type_*)(dest), 1, (type_*)(src), stride, len);                                                      \
+              CUDA_ERRCHK(cudaPeekAtLastError());                                                                                                                                            \
+              CUDA_ERRCHK(cudaStreamSynchronize(0));                                                                                                                                         \
+            }                                                                                                                                                                                \
+          }                                                                                                                                                                                  \
+        }
+
+#define AMPS_CONVERT_IN(type_, cvt, comm, src, dest, len, stride)                                                                                                                            \
+        {                                                                                                                                                                                    \
+          char *ptr_src, *ptr_dest;                                                                                                                                                          \
+          cudaPointerAttributes attributes;                                                                                                                                                  \
+          cudaPointerGetAttributes(&attributes, src);                                                                                                                                        \
+          if (cudaGetLastError() != cudaSuccess || attributes.type < 2) {                                                                                                                    \
+            if ((src) != (dest))                                                                                                                                                             \
+            {                                                                                                                                                                                \
+              if ((stride) == 1)                                                                                                                                                             \
+              {                                                                                                                                                                              \
+                bcopy((src), (dest), (size_t)(len) * sizeof(type_));                                                                                                                         \
+              }                                                                                                                                                                              \
+              else                                                                                                                                                                           \
+              {                                                                                                                                                                              \
+                for (ptr_src = (char*)(src), (ptr_dest) = (char*)(dest);                                                                                                                     \
+                     (ptr_dest) < (char*)(dest) + (size_t)((len) * (stride)) * sizeof(type_);                                                                                                \
+                     (ptr_src) += sizeof(type_), (ptr_dest) += sizeof(type_) * (size_t)((stride)))                                                                                           \
+                bcopy(ptr_src, ptr_dest, sizeof(type_));                                                                                                                                     \
+              }                                                                                                                                                                              \
+            }                                                                                                                                                                                \
+          }else{                                                                                                                                                                             \
+            if ((src) != (dest))                                                                                                                                                             \
+            {                                                                                                                                                                                \
+              const int blocksize = 1024;                                                                                                                                                    \
+              StridedCopyKernel << < (len + blocksize - 1) / blocksize, blocksize >> > (                                                                                                     \
+                                                                                        (type_*)(dest), stride, (type_*)(src), 1, len);                                                      \
+              CUDA_ERRCHK(cudaPeekAtLastError());                                                                                                                                            \
+              CUDA_ERRCHK(cudaStreamSynchronize(0));                                                                                                                                         \
+            }                                                                                                                                                                                \
+          }                                                                                                                                                                                  \
+        }
+#else
+#define AMPS_CONVERT_OUT(type, cvt, comm, src, dest, len, stride)                                                    \
+        {                                                                                                            \
+          type *ptr_src, *ptr_dest;                                                                                  \
+          if ((char*)(src) != (char*)(dest))                                                                         \
+          if ((stride) == 1)                                                                                         \
+          bcopy((src), (dest), (len) * sizeof(type));                                                                \
+          else                                                                                                       \
+          for (ptr_src = (type*)(src), ptr_dest = (type*)(dest); ptr_src < (type*)(src) + (len) * (stride);          \
+               ptr_src += (stride), ptr_dest++)                                                                      \
+          bcopy((ptr_src), (ptr_dest), sizeof(type));                                                                \
+        }
+
+#define AMPS_CONVERT_IN(type, cvt, comm, src, dest, len, stride)                                                     \
+        {                                                                                                            \
+          char *ptr_src, *ptr_dest;                                                                                  \
+          if ((src) != (dest))                                                                                       \
+          {                                                                                                          \
+            if ((stride) == 1)                                                                                       \
+            {                                                                                                        \
+              bcopy((src), (dest), (size_t)(len) * sizeof(type));                                                    \
+            }                                                                                                        \
+            else                                                                                                     \
+            {                                                                                                        \
+              for (ptr_src = (char*)(src), (ptr_dest) = (char*)(dest);                                               \
+                   (ptr_dest) < (char*)(dest) + (size_t)((len) * (stride)) * sizeof(type);                           \
+                   (ptr_src) += sizeof(type), (ptr_dest) += sizeof(type) * (size_t)((stride)))                       \
+              bcopy(ptr_src, ptr_dest, sizeof(type));                                                                \
+              ;                                                                                                      \
+            }                                                                                                        \
+          }                                                                                                          \
+        }
 #endif
 
 #define AMPS_CALL_BYTE_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(char, ctohc, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(char, ctohc, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_CHAR_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(char, ctohc, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(char, ctohc, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_SHORT_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(short, ctohs, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(short, ctohs, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_INT_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(int, ctohi, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(int, ctohi, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_LONG_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(long, ctohl, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(long, ctohl, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_FLOAT_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(float, ctohf, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(float, ctohf, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_DOUBLE_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(double, ctohd, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(double, ctohd, (_comm), (_src), (_dest), (_len), (_stride))
 
 
 #define AMPS_CALL_BYTE_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(char, htocc, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(char, htocc, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_CHAR_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(char, htocc, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(char, htocc, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_SHORT_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(short, htocs, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(short, htocs, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_INT_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(int, htoci, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(int, htoci, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_LONG_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(long, htocl, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(long, htocl, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_FLOAT_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(float, htocf, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(float, htocf, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_DOUBLE_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(double, htocd, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(double, htocd, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CHECK_OVERLAY(_type, _comm) 0
 
 #define AMPS_BYTE_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(char, _comm)
+        AMPS_CHECK_OVERLAY(char, _comm)
 
 #define AMPS_CHAR_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(char, _comm)
+        AMPS_CHECK_OVERLAY(char, _comm)
 
 #define AMPS_SHORT_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(short, _comm)
+        AMPS_CHECK_OVERLAY(short, _comm)
 
 #define AMPS_INT_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(int, _comm)
+        AMPS_CHECK_OVERLAY(int, _comm)
 
 #define AMPS_LONG_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(long, _comm)
+        AMPS_CHECK_OVERLAY(long, _comm)
 
 #define AMPS_FLOAT_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(float, _comm)
+        AMPS_CHECK_OVERLAY(float, _comm)
 
 #define AMPS_DOUBLE_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(double, _comm)
+        AMPS_CHECK_OVERLAY(double, _comm)
 
 /*---------------------------------------------------------------------------*/
 /* Macros for Invoice creation and deletion.                                 */
@@ -621,20 +619,20 @@ extern amps_Buffer *amps_BufferFreeList;
 /* Internal macros used to clear buffer and letter spaces.                   */
 /*---------------------------------------------------------------------------*/
 
-#define AMPS_CLEAR_INVOICE(invoice)                     \
-  {                                                     \
-    (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED; \
-    amps_ClearInvoice(invoice);                         \
-  }
-
-#define AMPS_PACK_FREE_LETTER(comm, invoice, amps_letter) \
-  if ((invoice)->combuf_flags & AMPS_INVOICE_OVERLAYED)   \
-    (invoice)->combuf_flags |= AMPS_INVOICE_ALLOCATED;    \
-  else                                                    \
-  {                                                       \
-    (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED;   \
-    amps_free((comm), (amps_letter));                     \
-  }                                                       \
+#define AMPS_CLEAR_INVOICE(invoice)                           \
+        {                                                     \
+          (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED; \
+          amps_ClearInvoice(invoice);                         \
+        }
+
+#define AMPS_PACK_FREE_LETTER(comm, invoice, amps_letter)       \
+        if ((invoice)->combuf_flags & AMPS_INVOICE_OVERLAYED)   \
+        (invoice)->combuf_flags |= AMPS_INVOICE_ALLOCATED;      \
+        else                                                    \
+        {                                                       \
+          (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED;   \
+          amps_free((comm), (amps_letter));                     \
+        }                                                       \
 
 /**
  *
@@ -938,28 +936,28 @@ extern amps_Buffer *amps_BufferFreeList;
 /*---------------------------------------------------------------------------*/
 
 #define amps_WriteChar(file, ptr, len) \
-  fwrite((ptr), sizeof(char), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_WriteShort(file, ptr, len) \
-  fwrite((ptr), sizeof(short), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(short), (len), (FILE*)(file))
 
 void amps_WriteInt(amps_File file, int *ptr, int len);
 
 #define amps_WriteLong(file, ptr, len) \
-  fwrite((ptr), sizeof(long), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(long), (len), (FILE*)(file))
 
 void amps_WriteDouble(amps_File file, double *ptr, int len);
 
 #define amps_ReadChar(file, ptr, len) \
-  fread((ptr), sizeof(char), (len), (FILE*)(file))
+        fread((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_ReadShort(file, ptr, len) \
-  fread((ptr), sizeof(short), (len), (FILE*)(file))
+        fread((ptr), sizeof(short), (len), (FILE*)(file))
 
 void amps_ReadInt(amps_File file, int *ptr, int len);
 
 #define amps_ReadLong(file, ptr, len) \
-  fread((ptr), sizeof(long), (len), (FILE*)(file))
+        fread((ptr), sizeof(long), (len), (FILE*)(file))
 
 void amps_ReadDouble(amps_File file, double *ptr, int len);
 
@@ -968,74 +966,74 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
 #ifdef AMPS_INTS_ARE_64
 
 #define amps_WriteChar(file, ptr, len) \
-  fwrite((ptr), sizeof(char), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_WriteShort(file, ptr, len) \
-  fwrite((ptr), sizeof(short), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(short), (len), (FILE*)(file))
 
 #define amps_WriteLong(file, ptr, len) \
-  fwrite((ptr), sizeof(long), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(long), (len), (FILE*)(file))
 
 #define amps_WriteFloat(file, ptr, len) \
-  fwrite((ptr), sizeof(float), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(float), (len), (FILE*)(file))
 
 #define amps_WriteDouble(file, ptr, len) \
-  fwrite((ptr), sizeof(double), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(double), (len), (FILE*)(file))
 
 
 #define amps_ReadChar(file, ptr, len) \
-  fread((ptr), sizeof(char), (len), (FILE*)(file))
+        fread((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_ReadShort(file, ptr, len) \
-  fread((ptr), sizeof(short), (len), (FILE*)(file))
+        fread((ptr), sizeof(short), (len), (FILE*)(file))
 
 #define amps_ReadLong(file, ptr, len) \
-  fread((ptr), sizeof(long), (len), (FILE*)(file))
+        fread((ptr), sizeof(long), (len), (FILE*)(file))
 
 #define amps_ReadFloat(file, ptr, len) \
-  fread((ptr), sizeof(float), (len), (FILE*)(file))
+        fread((ptr), sizeof(float), (len), (FILE*)(file))
 
 #define amps_ReadDouble(file, ptr, len) \
-  fread((ptr), sizeof(double), (len), (FILE*)(file))
+        fread((ptr), sizeof(double), (len), (FILE*)(file))
 
 #else
 
 #define amps_WriteChar(file, ptr, len) \
-  fwrite((ptr), sizeof(char), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_WriteShort(file, ptr, len) \
-  fwrite((ptr), sizeof(short), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(short), (len), (FILE*)(file))
 
 #define amps_WriteInt(file, ptr, len) \
-  fwrite((ptr), sizeof(int), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(int), (len), (FILE*)(file))
 
 #define amps_WriteLong(file, ptr, len) \
-  fwrite((ptr), sizeof(long), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(long), (len), (FILE*)(file))
 
 #define amps_WriteFloat(file, ptr, len) \
-  fwrite((ptr), sizeof(float), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(float), (len), (FILE*)(file))
 
 #define amps_WriteDouble(file, ptr, len) \
-  fwrite((ptr), sizeof(double), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(double), (len), (FILE*)(file))
 
 
 #define amps_ReadChar(file, ptr, len) \
-  fread((ptr), sizeof(char), (len), (FILE*)(file))
+        fread((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_ReadShort(file, ptr, len) \
-  fread((ptr), sizeof(short), (len), (FILE*)(file))
+        fread((ptr), sizeof(short), (len), (FILE*)(file))
 
 #define amps_ReadInt(file, ptr, len) \
-  fread((ptr), sizeof(int), (len), (FILE*)(file))
+        fread((ptr), sizeof(int), (len), (FILE*)(file))
 
 #define amps_ReadLong(file, ptr, len) \
-  fread((ptr), sizeof(long), (len), (FILE*)(file))
+        fread((ptr), sizeof(long), (len), (FILE*)(file))
 
 #define amps_ReadFloat(file, ptr, len) \
-  fread((ptr), sizeof(float), (len), (FILE*)(file))
+        fread((ptr), sizeof(float), (len), (FILE*)(file))
 
 #define amps_ReadDouble(file, ptr, len) \
-  fread((ptr), sizeof(double), (len), (FILE*)(file))
+        fread((ptr), sizeof(double), (len), (FILE*)(file))
 
 #endif
 #endif
@@ -1046,34 +1044,38 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
 
 /**
  * @brief CUDA error handling.
- * 
+ *
  * If error detected, print error message and exit.
  *
  * @param expr CUDA error (of type cudaError_t) [IN]
  */
-#define CUDA_ERRCHK( err ) (amps_cuda_error( err, __FILE__, __LINE__ ))
-static inline void amps_cuda_error(cudaError_t err, const char *file, int line) {
-	if (err != cudaSuccess) {
-		printf("\n\n%s in %s at line %d\n", cudaGetErrorString(err), file, line);
-		exit(1);
-	}
+#define CUDA_ERRCHK(err) (amps_cuda_error(err, __FILE__, __LINE__))
+static inline void amps_cuda_error(cudaError_t err, const char *file, int line)
+{
+  if (err != cudaSuccess)
+  {
+    printf("\n\n%s in %s at line %d\n", cudaGetErrorString(err), file, line);
+    exit(1);
+  }
 }
 
 #ifdef PARFLOW_HAVE_RMM
 #include <rmm/rmm_api.h>
 /**
  * @brief RMM error handling.
- * 
+ *
  * If error detected, print error message and exit.
  *
  * @param expr RMM error (of type rmmError_t) [IN]
  */
-#define RMM_ERRCHK( err ) (amps_rmm_error( err, __FILE__, __LINE__ ))
-static inline void amps_rmm_error(rmmError_t err, const char *file, int line) {
-	if (err != RMM_SUCCESS) {
-		printf("\n\n%s in %s at line %d\n", rmmGetErrorString(err), file, line);
-		exit(1);
-	}
+#define RMM_ERRCHK(err) (amps_rmm_error(err, __FILE__, __LINE__))
+static inline void amps_rmm_error(rmmError_t err, const char *file, int line)
+{
+  if (err != RMM_SUCCESS)
+  {
+    printf("\n\n%s in %s at line %d\n", rmmGetErrorString(err), file, line);
+    exit(1);
+  }
 }
 #endif
 
@@ -1084,63 +1086,65 @@ static inline void amps_rmm_error(rmmError_t err, const char *file, int line) {
 
 #ifdef __CUDACC__
 
-extern "C++"{
-template <typename T>
-__global__ static void 
+extern "C++" {
+template < typename T >
+__global__ static void
 __launch_bounds__(BLOCKSIZE_MAX)
-StridedCopyKernel(T * __restrict__ dest, const int stride_dest, 
-                                  T * __restrict__ src, const int stride_src, const int len) 
+StridedCopyKernel(T * __restrict__ dest, const int stride_dest,
+                  T * __restrict__ src, const int stride_src, const int len)
 {
-  const int tid = ((blockIdx.x*blockDim.x)+threadIdx.x);
-    
-  if(tid < len)
-  { 
+  const int tid = ((blockIdx.x * blockDim.x) + threadIdx.x);
+
+  if (tid < len)
+  {
     const int idx_dest = tid * stride_dest;
     const int idx_src = tid * stride_src;
 
     dest[idx_dest] = src[idx_src];
   }
 }
-template <typename T>
-__global__ static void 
+template < typename T >
+__global__ static void
 __launch_bounds__(BLOCKSIZE_MAX)
-PackingKernel(T * __restrict__ ptr_buf, const T * __restrict__ ptr_data, 
-    const int len_x, const int len_y, const int len_z, const int stride_x, const int stride_y, const int stride_z) 
+PackingKernel(T * __restrict__ ptr_buf, const T * __restrict__ ptr_data,
+              const int len_x, const int len_y, const int len_z, const int stride_x, const int stride_y, const int stride_z)
 {
-  const int k = ((blockIdx.z*blockDim.z)+threadIdx.z);   
-  if(k < len_z)
+  const int k = ((blockIdx.z * blockDim.z) + threadIdx.z);
+
+  if (k < len_z)
   {
-    const int j = ((blockIdx.y*blockDim.y)+threadIdx.y);   
-    if(j < len_y)
+    const int j = ((blockIdx.y * blockDim.y) + threadIdx.y);
+    if (j < len_y)
     {
-      const int i = ((blockIdx.x*blockDim.x)+threadIdx.x);   
-      if(i < len_x)
+      const int i = ((blockIdx.x * blockDim.x) + threadIdx.x);
+      if (i < len_x)
       {
-        *(ptr_buf + k * len_y * len_x + j * len_x + i) = 
-          *(ptr_data + k * (stride_z + (len_y - 1) * stride_y + len_y * (len_x - 1) * stride_x) + 
+        *(ptr_buf + k * len_y * len_x + j * len_x + i) =
+          *(ptr_data + k * (stride_z + (len_y - 1) * stride_y + len_y * (len_x - 1) * stride_x) +
             j * (stride_y + (len_x - 1) * stride_x) + i * stride_x);
       }
     }
   }
 }
-template <typename T>
-__global__ static void 
+template < typename T >
+__global__ static void
 __launch_bounds__(BLOCKSIZE_MAX)
-UnpackingKernel(const T * __restrict__ ptr_buf, T * __restrict__  ptr_data, 
-    const int len_x, const int len_y, const int len_z, const int stride_x, const int stride_y, const int stride_z) 
+UnpackingKernel(const T * __restrict__ ptr_buf, T * __restrict__ ptr_data,
+                const int len_x, const int len_y, const int len_z, const int stride_x, const int stride_y, const int stride_z)
 {
-  const int k = ((blockIdx.z*blockDim.z)+threadIdx.z);   
-  if(k < len_z)
+  const int k = ((blockIdx.z * blockDim.z) + threadIdx.z);
+
+  if (k < len_z)
   {
-    const int j = ((blockIdx.y*blockDim.y)+threadIdx.y);   
-    if(j < len_y)
+    const int j = ((blockIdx.y * blockDim.y) + threadIdx.y);
+    if (j < len_y)
     {
-      const int i = ((blockIdx.x*blockDim.x)+threadIdx.x);   
-      if(i < len_x)
+      const int i = ((blockIdx.x * blockDim.x) + threadIdx.x);
+      if (i < len_x)
       {
-        *(ptr_data + k * (stride_z + (len_y - 1) * stride_y + len_y * (len_x - 1) * stride_x) + 
-          j * (stride_y + (len_x - 1) * stride_x) + i * stride_x) = 
-            *(ptr_buf + k * len_y * len_x + j * len_x + i);
+        *(ptr_data + k * (stride_z + (len_y - 1) * stride_y + len_y * (len_x - 1) * stride_x) +
+          j * (stride_y + (len_x - 1) * stride_x) + i * stride_x) =
+          *(ptr_buf + k * len_y * len_x + j * len_x + i);
       }
     }
   }
@@ -1154,9 +1158,9 @@ UnpackingKernel(const T * __restrict__ ptr_buf, T * __restrict__  ptr_data,
 
 /**
  * @brief Allocates unified memory.
- * 
+ *
  * If RMM library is available, pool allocation is used for better performance.
- * 
+ *
  * @note Should not be called directly.
  *
  * @param size bytes to be allocated [IN]
@@ -1164,23 +1168,23 @@ UnpackingKernel(const T * __restrict__ ptr_buf, T * __restrict__  ptr_data,
  */
 static inline void *_amps_talloc_cuda(size_t size)
 {
-  void *ptr = NULL;  
-  
+  void *ptr = NULL;
+
 #ifdef PARFLOW_HAVE_RMM
-  RMM_ERRCHK(rmmAlloc(&ptr,size,0,__FILE__,__LINE__));
+  RMM_ERRCHK(rmmAlloc(&ptr, size, 0, __FILE__, __LINE__));
 #else
   CUDA_ERRCHK(cudaMallocManaged((void**)&ptr, size, cudaMemAttachGlobal));
-  // CUDA_ERRCHK(cudaHostAlloc((void**)&ptr, size, cudaHostAllocMapped));  
+  // CUDA_ERRCHK(cudaHostAlloc((void**)&ptr, size, cudaHostAllocMapped));
 #endif
-  
+
   return ptr;
 }
 
 /**
  * @brief Allocates unified memory initialized to 0.
- * 
+ *
  * If RMM library is available, pool allocation is used for better performance.
- * 
+ *
  * @note Should not be called directly.
  *
  * @param size bytes to be allocated [IN]
@@ -1188,23 +1192,23 @@ static inline void *_amps_talloc_cuda(size_t size)
  */
 static inline void *_amps_ctalloc_cuda(size_t size)
 {
-  void *ptr = NULL;  
+  void *ptr = NULL;
 
 #ifdef PARFLOW_HAVE_RMM
-  RMM_ERRCHK(rmmAlloc(&ptr,size,0,__FILE__,__LINE__));
+  RMM_ERRCHK(rmmAlloc(&ptr, size, 0, __FILE__, __LINE__));
 #else
   CUDA_ERRCHK(cudaMallocManaged((void**)&ptr, size, cudaMemAttachGlobal));
   // CUDA_ERRCHK(cudaHostAlloc((void**)&ptr, size, cudaHostAllocMapped));
-#endif  
+#endif
   // memset(ptr, 0, size);
-  CUDA_ERRCHK(cudaMemset(ptr, 0, size));  
-  
+  CUDA_ERRCHK(cudaMemset(ptr, 0, size));
+
   return ptr;
 }
 
 /**
  * @brief Frees unified memory allocated with \ref _talloc_cuda or \ref _ctalloc_cuda.
- * 
+ *
  * @note Should not be called directly.
  *
  * @param ptr a void pointer to the allocated dataspace [IN]
@@ -1212,7 +1216,7 @@ static inline void *_amps_ctalloc_cuda(size_t size)
 static inline void _amps_tfree_cuda(void *ptr)
 {
 #ifdef PARFLOW_HAVE_RMM
-  RMM_ERRCHK(rmmFree(ptr,0,__FILE__,__LINE__));
+  RMM_ERRCHK(rmmFree(ptr, 0, __FILE__, __LINE__));
 #else
   CUDA_ERRCHK(cudaFree(ptr));
   // CUDA_ERRCHK(cudaFreeHost(ptr));
@@ -1239,10 +1243,10 @@ static inline void _amps_tfree_cuda(void *ptr)
  * @return Pointer to the allocated dataspace
  */
 
-#define amps_TAlloc(type, count) ((count>0) ? (type*)_amps_talloc_cuda((unsigned int)(sizeof(type) * (count))) : NULL)
+#define amps_TAlloc(type, count) ((count > 0) ? (type*)_amps_talloc_cuda((unsigned int)(sizeof(type) * (count))) : NULL)
 
 /** Same as \ref amps_TAlloc for amps cuda layer */
-#define amps_TAlloc_managed(type, count) ((count>0) ? (type*)_amps_talloc_cuda((unsigned int)(sizeof(type) * (count))) : NULL)
+#define amps_TAlloc_managed(type, count) ((count > 0) ? (type*)_amps_talloc_cuda((unsigned int)(sizeof(type) * (count))) : NULL)
 
 /*===========================================================================*/
 /**
@@ -1294,7 +1298,7 @@ static inline void _amps_tfree_cuda(void *ptr)
 
 // SGS FIXME this should do something more than this
 #define amps_Error(name, type, comment, operation) \
-  printf("%s : %s\n", name, comment)
+        printf("%s : %s\n", name, comment)
 
 #include "amps_proto.h"
 
diff --git a/pfsimulator/amps/cuda/amps_allreduce.c b/pfsimulator/amps/cuda/amps_allreduce.c
index 5fd98c89c..ce9a4404c 100644
--- a/pfsimulator/amps/cuda/amps_allreduce.c
+++ b/pfsimulator/amps/cuda/amps_allreduce.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -102,7 +102,7 @@ int amps_AllReduce(amps_Comm comm, amps_Invoice invoice, MPI_Op operation)
     switch (ptr->type)
     {
       case AMPS_INVOICE_BYTE_CTYPE:
-	      mpi_type = MPI_BYTE;
+        mpi_type = MPI_BYTE;
         element_size = sizeof(char);
         break;
 
@@ -147,7 +147,8 @@ int amps_AllReduce(amps_Comm comm, amps_Invoice invoice, MPI_Op operation)
     struct cudaPointerAttributes attributes;
     cudaPointerGetAttributes(&attributes, (void *)data);
 
-    if(cudaGetLastError() == cudaSuccess && attributes.type > 1){
+    if (cudaGetLastError() == cudaSuccess && attributes.type > 1)
+    {
       if (stride == 1)
         CUDA_ERRCHK(cudaMemPrefetchAsync(data, (size_t)len * element_size, cudaCpuDeviceId, 0));
       else
@@ -156,7 +157,7 @@ int amps_AllReduce(amps_Comm comm, amps_Invoice invoice, MPI_Op operation)
              ptr_src += stride * element_size)
           CUDA_ERRCHK(cudaMemPrefetchAsync(ptr_src, (size_t)element_size, cudaCpuDeviceId, 0));
 
-      CUDA_ERRCHK(cudaStreamSynchronize(0)); 
+      CUDA_ERRCHK(cudaStreamSynchronize(0));
     }
 
     /* Copy into a contigous buffer */
@@ -184,6 +185,6 @@ int amps_AllReduce(amps_Comm comm, amps_Invoice invoice, MPI_Op operation)
 
     ptr = ptr->next;
   }
-  
+
   return 0;
 }
diff --git a/pfsimulator/amps/cuda/amps_bcast.c b/pfsimulator/amps/cuda/amps_bcast.c
index b1d6e8698..e4341facf 100644
--- a/pfsimulator/amps/cuda/amps_bcast.c
+++ b/pfsimulator/amps/cuda/amps_bcast.c
@@ -1,38 +1,38 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
 #define vtor(node) \
-  (node > source ? node : (node) ? node - 1 : source)
+        (node > source ? node : (node) ? node - 1 : source)
 
 #define rtov(node) \
-  (node > source ? node : (node < source) ? node + 1 : 0)
+        (node > source ? node : (node < source) ? node + 1 : 0)
 
 /*===========================================================================*/
 /**
diff --git a/pfsimulator/amps/cuda/amps_clear.c b/pfsimulator/amps/cuda/amps_clear.c
index 96bd8ce81..5721aeee8 100644
--- a/pfsimulator/amps/cuda/amps_clear.c
+++ b/pfsimulator/amps/cuda/amps_clear.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <string.h>
 #include <stdarg.h>
 
diff --git a/pfsimulator/amps/cuda/amps_createinvoice.c b/pfsimulator/amps/cuda/amps_createinvoice.c
index 59235a4f3..92dd3f540 100644
--- a/pfsimulator/amps/cuda/amps_createinvoice.c
+++ b/pfsimulator/amps/cuda/amps_createinvoice.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <string.h>
 #include <stdarg.h>
@@ -101,7 +101,7 @@ int amps_CreateInvoice(amps_Comm comm, amps_Invoice inv)
         }
         cur_pos += AMPS_CALL_BYTE_SIZEOF(comm, cur_pos, NULL, len, stride);
         break;
-	
+
       case AMPS_INVOICE_CHAR_CTYPE:
         cur_pos += AMPS_CALL_CHAR_ALIGN(comm, NULL, cur_pos, len, stride);
         if (ptr->data_type == AMPS_INVOICE_POINTER)
diff --git a/pfsimulator/amps/cuda/amps_exchange.c b/pfsimulator/amps/cuda/amps_exchange.c
index b3a21e8eb..f653f1ec0 100644
--- a/pfsimulator/amps/cuda/amps_exchange.c
+++ b/pfsimulator/amps/cuda/amps_exchange.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -45,30 +45,31 @@ int _amps_send_sizes(amps_Package package, int **sizes)
   /* check to see if enough space is already allocated            */
   if (amps_device_globals.combuf_send_size < size_acc)
   {
-    if (amps_device_globals.combuf_send_size != 0) CUDA_ERRCHK(cudaFree(amps_device_globals.combuf_send));
+    if (amps_device_globals.combuf_send_size != 0)
+      CUDA_ERRCHK(cudaFree(amps_device_globals.combuf_send));
 
     CUDA_ERRCHK(cudaMalloc((void**)&amps_device_globals.combuf_send, size_acc));
     amps_device_globals.combuf_send_size = size_acc;
   }
 
-  // CUDA_ERRCHK(cudaStreamSynchronize(0)); 
+  // CUDA_ERRCHK(cudaStreamSynchronize(0));
 
   size_acc = 0;
   for (int i = 0; i < package->num_send; i++)
   {
     package->send_invoices[i]->combuf = &amps_device_globals.combuf_send[size_acc];
     amps_pack(amps_CommWorld, package->send_invoices[i], package->send_invoices[i]->combuf, &streams_hired);
-    
+
     MPI_Isend(&((*sizes)[i]), 1, MPI_INT, package->dest[i],
               0, amps_CommWorld,
               &(package->send_requests[i]));
     size_acc += (*sizes)[i];
   }
 
-  for(int i = 0; i < streams_hired; i++)
+  for (int i = 0; i < streams_hired; i++)
   {
-    if(i < amps_device_max_streams)
-      CUDA_ERRCHK(cudaStreamSynchronize(amps_device_globals.stream[i])); 
+    if (i < amps_device_max_streams)
+      CUDA_ERRCHK(cudaStreamSynchronize(amps_device_globals.stream[i]));
   }
 
   return(0);
@@ -79,20 +80,22 @@ int _amps_recv_sizes(amps_Package package)
   int size_acc;
 
   MPI_Status status;
+
   sizes = (int*)calloc(package->num_recv, sizeof(int));
 
   size_acc = 0;
   for (int i = 0; i < package->num_recv; i++)
   {
     MPI_Recv(&sizes[i], 1, MPI_INT, package->src[i], 0,
-          amps_CommWorld, &status);
+             amps_CommWorld, &status);
     size_acc += sizes[i];
   }
 
   /* check to see if enough space is already allocated            */
   if (amps_device_globals.combuf_recv_size < size_acc)
   {
-    if (amps_device_globals.combuf_recv_size != 0) CUDA_ERRCHK(cudaFree(amps_device_globals.combuf_recv));
+    if (amps_device_globals.combuf_recv_size != 0)
+      CUDA_ERRCHK(cudaFree(amps_device_globals.combuf_recv));
 
     CUDA_ERRCHK(cudaMalloc((void**)&(amps_device_globals.combuf_recv), size_acc));
     CUDA_ERRCHK(cudaMemset(amps_device_globals.combuf_recv, 0, size_acc));
@@ -132,20 +135,20 @@ void _amps_wait_exchange(amps_Handle handle)
         amps_unpack(amps_CommWorld, handle->package->recv_invoices[i],
                     (char *)handle->package->recv_invoices[i]->combuf, &streams_hired);
       }
-      for(int i = 0; i < streams_hired; i++)
+      for (int i = 0; i < streams_hired; i++)
       {
-        if(i < amps_device_max_streams)
-          CUDA_ERRCHK(cudaStreamSynchronize(amps_device_globals.stream[i])); 
+        if (i < amps_device_max_streams)
+          CUDA_ERRCHK(cudaStreamSynchronize(amps_device_globals.stream[i]));
       }
     }
     for (i = 0; i < handle->package->num_recv; i++)
     {
-      if(handle->package->recv_requests[i] != MPI_REQUEST_NULL)
+      if (handle->package->recv_requests[i] != MPI_REQUEST_NULL)
         MPI_Request_free(&(handle->package->recv_requests[i]));
     }
     for (i = 0; i < handle->package->num_send; i++)
     {
-      if(handle->package->send_requests[i] != MPI_REQUEST_NULL)
+      if (handle->package->send_requests[i] != MPI_REQUEST_NULL)
         MPI_Request_free(&(handle->package->send_requests[i]));
     }
   }
@@ -153,7 +156,6 @@ void _amps_wait_exchange(amps_Handle handle)
 
 amps_Handle amps_IExchangePackage(amps_Package package)
 {
-
   int i;
   int *send_sizes;
 
diff --git a/pfsimulator/amps/cuda/amps_finalize.c b/pfsimulator/amps/cuda/amps_finalize.c
index 5127f3e73..1ae8f4a43 100644
--- a/pfsimulator/amps/cuda/amps_finalize.c
+++ b/pfsimulator/amps/cuda/amps_finalize.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 extern int amps_mpi_initialized;
@@ -71,10 +71,13 @@ int amps_Finalize()
     MPI_Finalize();
   }
 
-  if (amps_device_globals.combuf_recv_size != 0) CUDA_ERRCHK(cudaFree(amps_device_globals.combuf_recv));
-  if (amps_device_globals.combuf_send_size != 0) CUDA_ERRCHK(cudaFree(amps_device_globals.combuf_send));
+  if (amps_device_globals.combuf_recv_size != 0)
+    CUDA_ERRCHK(cudaFree(amps_device_globals.combuf_recv));
+  if (amps_device_globals.combuf_send_size != 0)
+    CUDA_ERRCHK(cudaFree(amps_device_globals.combuf_send));
 
-  for(int i = 0; i < amps_device_globals.streams_created; i++){
+  for (int i = 0; i < amps_device_globals.streams_created; i++)
+  {
     CUDA_ERRCHK(cudaStreamDestroy(amps_device_globals.stream[i]));
   }
 
diff --git a/pfsimulator/amps/cuda/amps_init.c b/pfsimulator/amps/cuda/amps_init.c
index 52934cdb4..eb92536a7 100644
--- a/pfsimulator/amps/cuda/amps_init.c
+++ b/pfsimulator/amps/cuda/amps_init.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -52,9 +52,9 @@ MPI_Comm amps_CommWorld = MPI_COMM_NULL;
 MPI_Comm amps_CommNode = MPI_COMM_NULL;
 MPI_Comm amps_CommWrite = MPI_COMM_NULL;
 
-amps_Devicestruct amps_device_globals = {.combuf_recv_size = 0, 
-                                         .combuf_send_size = 0, 
-                                         .streams_created = 0};
+amps_Devicestruct amps_device_globals = { .combuf_recv_size = 0,
+                                          .combuf_send_size = 0,
+                                          .streams_created = 0 };
 
 #ifdef AMPS_F2CLIB_FIX
 int MAIN__()
@@ -120,7 +120,7 @@ int amps_Init(int *argc, char **argv[])
 
   MPI_Init(argc, argv);
   amps_mpi_initialized = TRUE;
-  
+
   MPI_Comm_dup(MPI_COMM_WORLD, &amps_CommWorld);
   MPI_Comm_size(amps_CommWorld, &amps_size);
   MPI_Comm_rank(amps_CommWorld, &amps_rank);
@@ -139,7 +139,7 @@ int amps_Init(int *argc, char **argv[])
   checkSum &= INT_MAX;
   MPI_Comm_split(amps_CommWorld, checkSum, amps_rank, &amps_CommNode);
 #endif
-  
+
   MPI_Comm_rank(amps_CommNode, &amps_node_rank);
   MPI_Comm_size(amps_CommNode, &amps_node_size);
   int color;
diff --git a/pfsimulator/amps/cuda/amps_invoice.c b/pfsimulator/amps/cuda/amps_invoice.c
index 23da9a1de..1fe98c983 100644
--- a/pfsimulator/amps/cuda/amps_invoice.c
+++ b/pfsimulator/amps/cuda/amps_invoice.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <string.h>
 #include <ctype.h>
diff --git a/pfsimulator/amps/cuda/amps_irecv.c b/pfsimulator/amps/cuda/amps_irecv.c
index e177ace95..39ae32ff3 100644
--- a/pfsimulator/amps/cuda/amps_irecv.c
+++ b/pfsimulator/amps/cuda/amps_irecv.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 /*===========================================================================*/
diff --git a/pfsimulator/amps/cuda/amps_newpackage.c b/pfsimulator/amps/cuda/amps_newpackage.c
index 911a88dc7..c9c4f07bb 100644
--- a/pfsimulator/amps/cuda/amps_newpackage.c
+++ b/pfsimulator/amps/cuda/amps_newpackage.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/cuda/amps_recv.c b/pfsimulator/amps/cuda/amps_recv.c
index b37fdc05e..2de3a4e8b 100644
--- a/pfsimulator/amps/cuda/amps_recv.c
+++ b/pfsimulator/amps/cuda/amps_recv.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/cuda/amps_send.c b/pfsimulator/amps/cuda/amps_send.c
index 2218bdfee..884939bea 100644
--- a/pfsimulator/amps/cuda/amps_send.c
+++ b/pfsimulator/amps/cuda/amps_send.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/cuda/amps_sizeofinvoice.c b/pfsimulator/amps/cuda/amps_sizeofinvoice.c
index 70684c289..2146229db 100644
--- a/pfsimulator/amps/cuda/amps_sizeofinvoice.c
+++ b/pfsimulator/amps/cuda/amps_sizeofinvoice.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -66,7 +66,7 @@ long amps_sizeof_invoice(
         cur_pos += AMPS_CALL_BYTE_SIZEOF(comm, data, cur_pos,
                                          len, stride);
         break;
-      
+
       case AMPS_INVOICE_CHAR_CTYPE:
         cur_pos += AMPS_CALL_CHAR_ALIGN(comm, data, cur_pos, len, stride);
         cur_pos += AMPS_CALL_CHAR_SIZEOF(comm, data, cur_pos,
diff --git a/pfsimulator/amps/cuda/amps_test.c b/pfsimulator/amps/cuda/amps_test.c
index 0cdf2df18..0338d7df7 100644
--- a/pfsimulator/amps/cuda/amps_test.c
+++ b/pfsimulator/amps/cuda/amps_test.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/cuda/amps_vector.c b/pfsimulator/amps/cuda/amps_vector.c
index 0105d4be8..347499e76 100644
--- a/pfsimulator/amps/cuda/amps_vector.c
+++ b/pfsimulator/amps/cuda/amps_vector.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 #include <strings.h>
diff --git a/pfsimulator/amps/cuda/peekmc b/pfsimulator/amps/cuda/peekmc
index 47f356a76..7cc6c6a84 100755
--- a/pfsimulator/amps/cuda/peekmc
+++ b/pfsimulator/amps/cuda/peekmc
@@ -1,7 +1,7 @@
 #! /bin/sh
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pfsimulator/amps/cuda/run b/pfsimulator/amps/cuda/run
index a9d8cf218..3b833be18 100755
--- a/pfsimulator/amps/cuda/run
+++ b/pfsimulator/amps/cuda/run
@@ -1,7 +1,7 @@
 #! /bin/sh
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pfsimulator/amps/mpi1/amps.h b/pfsimulator/amps/mpi1/amps.h
index 45406aabf..7040058a2 100644
--- a/pfsimulator/amps/mpi1/amps.h
+++ b/pfsimulator/amps/mpi1/amps.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #ifndef amps_include
 #define amps_include
 
@@ -384,160 +384,160 @@ extern amps_Buffer *amps_BufferFreeList;
 /*---------------------------------------------------------------------------*/
 /* Functions to for align                                                    */
 /*---------------------------------------------------------------------------*/
-#define AMPS_ALIGN(type, src, dest, len, stride) \
-  ((sizeof(type) -                               \
-    ((unsigned long)(dest) % sizeof(type)))      \
-   % sizeof(type));
+#define AMPS_ALIGN(type, src, dest, len, stride)       \
+        ((sizeof(type) -                               \
+          ((unsigned long)(dest) % sizeof(type)))      \
+         % sizeof(type));
 
 
 #define AMPS_CALL_BYTE_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(char, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(char, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_CHAR_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(char, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(char, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_SHORT_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(short, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(short, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_INT_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(int, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(int, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_LONG_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(long, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(long, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_FLOAT_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(float, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(float, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_DOUBLE_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(double, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(double, (_src), (_dest), (_len), (_stride))
 
 /*---------------------------------------------------------------------------*/
 /* Functions to for sizeof                                                   */
 /*---------------------------------------------------------------------------*/
 #define AMPS_SIZEOF(len, stride, size) \
-  (size_t)(len) * (size)
+        (size_t)(len) * (size)
 
-#define AMPS_CALL_BYTE_SIZEOF(_comm, _src, _dest, _len, _stride)	\
-  AMPS_SIZEOF((_len), (_stride), sizeof(char))
+#define AMPS_CALL_BYTE_SIZEOF(_comm, _src, _dest, _len, _stride)        \
+        AMPS_SIZEOF((_len), (_stride), sizeof(char))
 
 #define AMPS_CALL_CHAR_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(char))
+        AMPS_SIZEOF((_len), (_stride), sizeof(char))
 
 #define AMPS_CALL_SHORT_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(short))
+        AMPS_SIZEOF((_len), (_stride), sizeof(short))
 
 #define AMPS_CALL_INT_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(int))
+        AMPS_SIZEOF((_len), (_stride), sizeof(int))
 
 #define AMPS_CALL_LONG_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(long))
+        AMPS_SIZEOF((_len), (_stride), sizeof(long))
 
 #define AMPS_CALL_FLOAT_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(float))
+        AMPS_SIZEOF((_len), (_stride), sizeof(float))
 
 #define AMPS_CALL_DOUBLE_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(double))
+        AMPS_SIZEOF((_len), (_stride), sizeof(double))
 
 /*---------------------------------------------------------------------------*/
 /*---------------------------------------------------------------------------*/
-#define AMPS_CONVERT_OUT(type, cvt, comm, src, dest, len, stride)                                         \
-  {                                                                                                       \
-    type *ptr_src, *ptr_dest;                                                                             \
-    if ((char*)(src) != (char*)(dest))                                                                    \
-      if ((stride) == 1)                                                                                  \
-        bcopy((src), (dest), (len) * sizeof(type));                                                       \
-      else                                                                                                \
-        for (ptr_src = (type*)(src), ptr_dest = (type*)(dest); ptr_src < (type*)(src) + (len) * (stride); \
-             ptr_src += (stride), ptr_dest++)                                                             \
-          bcopy((ptr_src), (ptr_dest), sizeof(type));                                                     \
-  }
+#define AMPS_CONVERT_OUT(type, cvt, comm, src, dest, len, stride)                                               \
+        {                                                                                                       \
+          type *ptr_src, *ptr_dest;                                                                             \
+          if ((char*)(src) != (char*)(dest))                                                                    \
+          if ((stride) == 1)                                                                                    \
+          bcopy((src), (dest), (len) * sizeof(type));                                                           \
+          else                                                                                                  \
+          for (ptr_src = (type*)(src), ptr_dest = (type*)(dest); ptr_src < (type*)(src) + (len) * (stride);     \
+               ptr_src += (stride), ptr_dest++)                                                                 \
+          bcopy((ptr_src), (ptr_dest), sizeof(type));                                                           \
+        }
 
 #define AMPS_CALL_BYTE_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(char, ctohc, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(char, ctohc, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_CHAR_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(char, ctohc, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(char, ctohc, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_SHORT_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(short, ctohs, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(short, ctohs, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_INT_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(int, ctohi, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(int, ctohi, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_LONG_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(long, ctohl, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(long, ctohl, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_FLOAT_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(float, ctohf, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(float, ctohf, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_DOUBLE_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(double, ctohd, (_comm), (_src), (_dest), (_len), (_stride))
-
-
-
-#define AMPS_CONVERT_IN(type, cvt, comm, src, dest, len, stride)                         \
-  {                                                                                      \
-    char *ptr_src, *ptr_dest;                                                            \
-    if ((src) != (dest))                                                                 \
-    {                                                                                    \
-      if ((stride) == 1)                                                                 \
-      {                                                                                  \
-        bcopy((src), (dest), (size_t)(len) * sizeof(type));                              \
-      }                                                                                  \
-      else                                                                               \
-      {                                                                                  \
-        for (ptr_src = (char*)(src), (ptr_dest) = (char*)(dest);                         \
-             (ptr_dest) < (char*)(dest) + (size_t)((len) * (stride)) * sizeof(type);     \
-             (ptr_src) += sizeof(type), (ptr_dest) += sizeof(type) * (size_t)((stride))) \
-          bcopy(ptr_src, ptr_dest, sizeof(type));                                        \
-        ;                                                                                \
-      }                                                                                  \
-    }                                                                                    \
-  }
+        AMPS_CONVERT_OUT(double, ctohd, (_comm), (_src), (_dest), (_len), (_stride))
+
+
+
+#define AMPS_CONVERT_IN(type, cvt, comm, src, dest, len, stride)                               \
+        {                                                                                      \
+          char *ptr_src, *ptr_dest;                                                            \
+          if ((src) != (dest))                                                                 \
+          {                                                                                    \
+            if ((stride) == 1)                                                                 \
+            {                                                                                  \
+              bcopy((src), (dest), (size_t)(len) * sizeof(type));                              \
+            }                                                                                  \
+            else                                                                               \
+            {                                                                                  \
+              for (ptr_src = (char*)(src), (ptr_dest) = (char*)(dest);                         \
+                   (ptr_dest) < (char*)(dest) + (size_t)((len) * (stride)) * sizeof(type);     \
+                   (ptr_src) += sizeof(type), (ptr_dest) += sizeof(type) * (size_t)((stride))) \
+              bcopy(ptr_src, ptr_dest, sizeof(type));                                          \
+              ;                                                                                \
+            }                                                                                  \
+          }                                                                                    \
+        }
 
 #define AMPS_CALL_BYTE_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(char, htocc, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(char, htocc, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_CHAR_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(char, htocc, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(char, htocc, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_SHORT_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(short, htocs, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(short, htocs, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_INT_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(int, htoci, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(int, htoci, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_LONG_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(long, htocl, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(long, htocl, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_FLOAT_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(float, htocf, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(float, htocf, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_DOUBLE_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(double, htocd, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(double, htocd, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CHECK_OVERLAY(_type, _comm) 0
 
 #define AMPS_BYTE_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(char, _comm)
+        AMPS_CHECK_OVERLAY(char, _comm)
 
 #define AMPS_CHAR_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(char, _comm)
+        AMPS_CHECK_OVERLAY(char, _comm)
 
 #define AMPS_SHORT_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(short, _comm)
+        AMPS_CHECK_OVERLAY(short, _comm)
 
 #define AMPS_INT_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(int, _comm)
+        AMPS_CHECK_OVERLAY(int, _comm)
 
 #define AMPS_LONG_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(long, _comm)
+        AMPS_CHECK_OVERLAY(long, _comm)
 
 #define AMPS_FLOAT_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(float, _comm)
+        AMPS_CHECK_OVERLAY(float, _comm)
 
 #define AMPS_DOUBLE_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(double, _comm)
+        AMPS_CHECK_OVERLAY(double, _comm)
 
 /*---------------------------------------------------------------------------*/
 /* Macros for Invoice creation and deletion.                                 */
@@ -549,20 +549,20 @@ extern amps_Buffer *amps_BufferFreeList;
 /* Internal macros used to clear buffer and letter spaces.                   */
 /*---------------------------------------------------------------------------*/
 
-#define AMPS_CLEAR_INVOICE(invoice)                     \
-  {                                                     \
-    (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED; \
-    amps_ClearInvoice(invoice);                         \
-  }
-
-#define AMPS_PACK_FREE_LETTER(comm, invoice, amps_letter) \
-  if ((invoice)->combuf_flags & AMPS_INVOICE_OVERLAYED)   \
-    (invoice)->combuf_flags |= AMPS_INVOICE_ALLOCATED;    \
-  else                                                    \
-  {                                                       \
-    (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED;   \
-    amps_free((comm), (amps_letter));                     \
-  }                                                       \
+#define AMPS_CLEAR_INVOICE(invoice)                           \
+        {                                                     \
+          (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED; \
+          amps_ClearInvoice(invoice);                         \
+        }
+
+#define AMPS_PACK_FREE_LETTER(comm, invoice, amps_letter)       \
+        if ((invoice)->combuf_flags & AMPS_INVOICE_OVERLAYED)   \
+        (invoice)->combuf_flags |= AMPS_INVOICE_ALLOCATED;      \
+        else                                                    \
+        {                                                       \
+          (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED;   \
+          amps_free((comm), (amps_letter));                     \
+        }                                                       \
 
 /**
  *
@@ -866,28 +866,28 @@ extern amps_Buffer *amps_BufferFreeList;
 /*---------------------------------------------------------------------------*/
 
 #define amps_WriteChar(file, ptr, len) \
-  fwrite((ptr), sizeof(char), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_WriteShort(file, ptr, len) \
-  fwrite((ptr), sizeof(short), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(short), (len), (FILE*)(file))
 
 void amps_WriteInt(amps_File file, int *ptr, int len);
 
 #define amps_WriteLong(file, ptr, len) \
-  fwrite((ptr), sizeof(long), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(long), (len), (FILE*)(file))
 
 void amps_WriteDouble(amps_File file, double *ptr, int len);
 
 #define amps_ReadChar(file, ptr, len) \
-  fread((ptr), sizeof(char), (len), (FILE*)(file))
+        fread((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_ReadShort(file, ptr, len) \
-  fread((ptr), sizeof(short), (len), (FILE*)(file))
+        fread((ptr), sizeof(short), (len), (FILE*)(file))
 
 void amps_ReadInt(amps_File file, int *ptr, int len);
 
 #define amps_ReadLong(file, ptr, len) \
-  fread((ptr), sizeof(long), (len), (FILE*)(file))
+        fread((ptr), sizeof(long), (len), (FILE*)(file))
 
 void amps_ReadDouble(amps_File file, double *ptr, int len);
 
@@ -896,74 +896,74 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
 #ifdef AMPS_INTS_ARE_64
 
 #define amps_WriteChar(file, ptr, len) \
-  fwrite((ptr), sizeof(char), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_WriteShort(file, ptr, len) \
-  fwrite((ptr), sizeof(short), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(short), (len), (FILE*)(file))
 
 #define amps_WriteLong(file, ptr, len) \
-  fwrite((ptr), sizeof(long), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(long), (len), (FILE*)(file))
 
 #define amps_WriteFloat(file, ptr, len) \
-  fwrite((ptr), sizeof(float), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(float), (len), (FILE*)(file))
 
 #define amps_WriteDouble(file, ptr, len) \
-  fwrite((ptr), sizeof(double), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(double), (len), (FILE*)(file))
 
 
 #define amps_ReadChar(file, ptr, len) \
-  fread((ptr), sizeof(char), (len), (FILE*)(file))
+        fread((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_ReadShort(file, ptr, len) \
-  fread((ptr), sizeof(short), (len), (FILE*)(file))
+        fread((ptr), sizeof(short), (len), (FILE*)(file))
 
 #define amps_ReadLong(file, ptr, len) \
-  fread((ptr), sizeof(long), (len), (FILE*)(file))
+        fread((ptr), sizeof(long), (len), (FILE*)(file))
 
 #define amps_ReadFloat(file, ptr, len) \
-  fread((ptr), sizeof(float), (len), (FILE*)(file))
+        fread((ptr), sizeof(float), (len), (FILE*)(file))
 
 #define amps_ReadDouble(file, ptr, len) \
-  fread((ptr), sizeof(double), (len), (FILE*)(file))
+        fread((ptr), sizeof(double), (len), (FILE*)(file))
 
 #else
 
 #define amps_WriteChar(file, ptr, len) \
-  fwrite((ptr), sizeof(char), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_WriteShort(file, ptr, len) \
-  fwrite((ptr), sizeof(short), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(short), (len), (FILE*)(file))
 
 #define amps_WriteInt(file, ptr, len) \
-  fwrite((ptr), sizeof(int), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(int), (len), (FILE*)(file))
 
 #define amps_WriteLong(file, ptr, len) \
-  fwrite((ptr), sizeof(long), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(long), (len), (FILE*)(file))
 
 #define amps_WriteFloat(file, ptr, len) \
-  fwrite((ptr), sizeof(float), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(float), (len), (FILE*)(file))
 
 #define amps_WriteDouble(file, ptr, len) \
-  fwrite((ptr), sizeof(double), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(double), (len), (FILE*)(file))
 
 
 #define amps_ReadChar(file, ptr, len) \
-  fread((ptr), sizeof(char), (len), (FILE*)(file))
+        fread((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_ReadShort(file, ptr, len) \
-  fread((ptr), sizeof(short), (len), (FILE*)(file))
+        fread((ptr), sizeof(short), (len), (FILE*)(file))
 
 #define amps_ReadInt(file, ptr, len) \
-  fread((ptr), sizeof(int), (len), (FILE*)(file))
+        fread((ptr), sizeof(int), (len), (FILE*)(file))
 
 #define amps_ReadLong(file, ptr, len) \
-  fread((ptr), sizeof(long), (len), (FILE*)(file))
+        fread((ptr), sizeof(long), (len), (FILE*)(file))
 
 #define amps_ReadFloat(file, ptr, len) \
-  fread((ptr), sizeof(float), (len), (FILE*)(file))
+        fread((ptr), sizeof(float), (len), (FILE*)(file))
 
 #define amps_ReadDouble(file, ptr, len) \
-  fread((ptr), sizeof(double), (len), (FILE*)(file))
+        fread((ptr), sizeof(double), (len), (FILE*)(file))
 
 #endif
 #endif
@@ -987,7 +987,7 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
  * @param count Number of items of type to allocate
  * @return Pointer to the allocated dataspace
  */
-#define amps_TAlloc(type, count) ((count>0) ? (type*)malloc((unsigned int)(sizeof(type) * (count))) : NULL)
+#define amps_TAlloc(type, count) ((count > 0) ? (type*)malloc((unsigned int)(sizeof(type) * (count))) : NULL)
 
 /*===========================================================================*/
 /**
@@ -1011,7 +1011,7 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
  */
 
 #define amps_CTAlloc(type, count) \
-  ((count) ? (type*)calloc((unsigned int)(count), (unsigned int)sizeof(type)) : NULL)
+        ((count) ? (type*)calloc((unsigned int)(count), (unsigned int)sizeof(type)) : NULL)
 
 /**
  *
@@ -1033,7 +1033,7 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
 
 // SGS FIXME this should do something more than this
 #define amps_Error(name, type, comment, operation) \
-  printf("%s : %s\n", name, comment)
+        printf("%s : %s\n", name, comment)
 
 #if defined(PARFLOW_HAVE_CUDA) || defined(PARFLOW_HAVE_KOKKOS)
 /*--------------------------------------------------------------------------
@@ -1042,9 +1042,9 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
 
 /**
  * @brief Operation modes for amps_gpupacking function
- *  
+ *
  * @note See function description for amps_gpupacking.
- * 
+ *
  * @{
  */
 #define AMPS_GETRBUF 1
@@ -1065,34 +1065,38 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
 
 /**
  * @brief CUDA error handling
- * 
+ *
  * If error detected, print error message and exit.
  *
  * @param expr CUDA error (of type cudaError_t) [IN]
  */
-#define CUDA_ERRCHK( err ) (amps_cuda_error( err, __FILE__, __LINE__ ))
-static inline void amps_cuda_error(cudaError_t err, const char *file, int line) {
-	if (err != cudaSuccess) {
-		printf("\n\n%s in %s at line %d\n", cudaGetErrorString(err), file, line);
-		exit(1);
-	}
+#define CUDA_ERRCHK(err) (amps_cuda_error(err, __FILE__, __LINE__))
+static inline void amps_cuda_error(cudaError_t err, const char *file, int line)
+{
+  if (err != cudaSuccess)
+  {
+    printf("\n\n%s in %s at line %d\n", cudaGetErrorString(err), file, line);
+    exit(1);
+  }
 }
 #endif // PARFLOW_HAVE_CUDA
 
 #ifdef PARFLOW_HAVE_RMM
 /**
  * @brief RMM error handling
- * 
+ *
  * If error detected, print error message and exit.
  *
  * @param expr RMM error (of type rmmError_t) [IN]
  */
-#define RMM_ERRCHK( err ) (amps_rmm_error( err, __FILE__, __LINE__ ))
-static inline void amps_rmm_error(rmmError_t err, const char *file, int line) {
-	if (err != RMM_SUCCESS) {
-		printf("\n\n%s in %s at line %d\n", rmmGetErrorString(err), file, line);
-		exit(1);
-	}
+#define RMM_ERRCHK(err) (amps_rmm_error(err, __FILE__, __LINE__))
+static inline void amps_rmm_error(rmmError_t err, const char *file, int line)
+{
+  if (err != RMM_SUCCESS)
+  {
+    printf("\n\n%s in %s at line %d\n", rmmGetErrorString(err), file, line);
+    exit(1);
+  }
 }
 #endif // PARFLOW_HAVE_RMM
 
@@ -1122,9 +1126,9 @@ void kokkosMemSetAmps(char *ptr, size_t size);
 
 /**
  * @brief Allocates unified memory
- * 
+ *
  * If RMM library is available, pool allocation is used for better performance.
- * 
+ *
  * @note Should not be called directly.
  *
  * @param size bytes to be allocated [IN]
@@ -1132,25 +1136,25 @@ void kokkosMemSetAmps(char *ptr, size_t size);
  */
 static inline void *_amps_talloc_device(size_t size)
 {
-  void *ptr = NULL;  
-  
+  void *ptr = NULL;
+
 #ifdef PARFLOW_HAVE_RMM
-  RMM_ERRCHK(rmmAlloc(&ptr,size,0,__FILE__,__LINE__));
+  RMM_ERRCHK(rmmAlloc(&ptr, size, 0, __FILE__, __LINE__));
 #elif defined(PARFLOW_HAVE_KOKKOS)
   ptr = kokkosUVMAlloc(size);
 #elif defined(PARFLOW_HAVE_CUDA)
   CUDA_ERRCHK(cudaMallocManaged((void**)&ptr, size, cudaMemAttachGlobal));
-  // CUDA_ERRCHK(cudaHostAlloc((void**)&ptr, size, cudaHostAllocMapped));  
+  // CUDA_ERRCHK(cudaHostAlloc((void**)&ptr, size, cudaHostAllocMapped));
 #endif
-  
+
   return ptr;
 }
 
 /**
  * @brief Allocates unified memory initialized to 0
- * 
+ *
  * If RMM library is available, pool allocation is used for better performance.
- * 
+ *
  * @note Should not be called directly.
  *
  * @param size bytes to be allocated [IN]
@@ -1158,30 +1162,30 @@ static inline void *_amps_talloc_device(size_t size)
  */
 static inline void *_amps_ctalloc_device(size_t size)
 {
-  void *ptr = NULL;  
+  void *ptr = NULL;
 
 #ifdef PARFLOW_HAVE_RMM
-  RMM_ERRCHK(rmmAlloc(&ptr,size,0,__FILE__,__LINE__));
+  RMM_ERRCHK(rmmAlloc(&ptr, size, 0, __FILE__, __LINE__));
 #elif defined(PARFLOW_HAVE_KOKKOS)
   ptr = kokkosUVMAlloc(size);
 #elif defined(PARFLOW_HAVE_CUDA)
   CUDA_ERRCHK(cudaMallocManaged((void**)&ptr, size, cudaMemAttachGlobal));
   // CUDA_ERRCHK(cudaHostAlloc((void**)&ptr, size, cudaHostAllocMapped));
-#endif  
+#endif
 
 #if defined(PARFLOW_HAVE_CUDA)
-  CUDA_ERRCHK(cudaMemset(ptr, 0, size));  
+  CUDA_ERRCHK(cudaMemset(ptr, 0, size));
 #else
   // memset(ptr, 0, size);
   kokkosMemSetAmps((char*)ptr, size);
 #endif
-  
+
   return ptr;
 }
 
 /**
  * @brief Frees unified memory allocated with \ref _talloc_cuda or \ref _ctalloc_cuda
- * 
+ *
  * @note Should not be called directly.
  *
  * @param ptr a void pointer to the allocated dataspace [IN]
@@ -1189,7 +1193,7 @@ static inline void *_amps_ctalloc_device(size_t size)
 static inline void _amps_tfree_device(void *ptr)
 {
 #ifdef PARFLOW_HAVE_RMM
-  RMM_ERRCHK(rmmFree(ptr,0,__FILE__,__LINE__));
+  RMM_ERRCHK(rmmFree(ptr, 0, __FILE__, __LINE__));
 #elif defined(PARFLOW_HAVE_KOKKOS)
   kokkosUVMFree(ptr);
 #elif defined(PARFLOW_HAVE_CUDA)
@@ -1198,17 +1202,17 @@ static inline void _amps_tfree_device(void *ptr)
 #endif
 }
 
-/** 
+/**
  * Same as \ref amps_TAlloc but allocates managed memory
  */
-#define amps_TAlloc_managed(type, count) ((count>0) ? (type*)_amps_talloc_device((unsigned int)(sizeof(type) * (count))) : NULL)
+#define amps_TAlloc_managed(type, count) ((count > 0) ? (type*)_amps_talloc_device((unsigned int)(sizeof(type) * (count))) : NULL)
 
-/** 
+/**
  * Same as \ref amps_CTAlloc but allocates managed memory
  */
 #define amps_CTAlloc_managed(type, count) ((count) ? (type*)_amps_ctalloc_device((unsigned int)(sizeof(type) * (count))) : NULL)
 
-/** 
+/**
  * Same as \ref amps_TFree but deallocates managed memory
  */
 #define amps_TFree_managed(ptr) if (ptr) _amps_tfree_device(ptr); else {}
diff --git a/pfsimulator/amps/mpi1/amps_allreduce.c b/pfsimulator/amps/mpi1/amps_allreduce.c
index dc0ebab36..aeabc6c54 100644
--- a/pfsimulator/amps/mpi1/amps_allreduce.c
+++ b/pfsimulator/amps/mpi1/amps_allreduce.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -102,7 +102,7 @@ int amps_AllReduce(amps_Comm comm, amps_Invoice invoice, MPI_Op operation)
     switch (ptr->type)
     {
       case AMPS_INVOICE_BYTE_CTYPE:
-	      mpi_type = MPI_BYTE;
+        mpi_type = MPI_BYTE;
         element_size = sizeof(char);
         break;
 
@@ -148,7 +148,8 @@ int amps_AllReduce(amps_Comm comm, amps_Invoice invoice, MPI_Op operation)
     struct cudaPointerAttributes attributes;
     cudaPointerGetAttributes(&attributes, (void *)data);
 
-    if(cudaGetLastError() == cudaSuccess && attributes.type > 1){
+    if (cudaGetLastError() == cudaSuccess && attributes.type > 1)
+    {
       if (stride == 1)
         CUDA_ERRCHK(cudaMemPrefetchAsync(data, (size_t)len * element_size, cudaCpuDeviceId, 0));
       else
@@ -157,7 +158,7 @@ int amps_AllReduce(amps_Comm comm, amps_Invoice invoice, MPI_Op operation)
              ptr_src += stride * element_size)
           CUDA_ERRCHK(cudaMemPrefetchAsync(ptr_src, (size_t)element_size, cudaCpuDeviceId, 0));
 
-      CUDA_ERRCHK(cudaStreamSynchronize(0)); 
+      CUDA_ERRCHK(cudaStreamSynchronize(0));
     }
 #endif
 
diff --git a/pfsimulator/amps/mpi1/amps_bcast.c b/pfsimulator/amps/mpi1/amps_bcast.c
index b1d6e8698..e4341facf 100644
--- a/pfsimulator/amps/mpi1/amps_bcast.c
+++ b/pfsimulator/amps/mpi1/amps_bcast.c
@@ -1,38 +1,38 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
 #define vtor(node) \
-  (node > source ? node : (node) ? node - 1 : source)
+        (node > source ? node : (node) ? node - 1 : source)
 
 #define rtov(node) \
-  (node > source ? node : (node < source) ? node + 1 : 0)
+        (node > source ? node : (node < source) ? node + 1 : 0)
 
 /*===========================================================================*/
 /**
diff --git a/pfsimulator/amps/mpi1/amps_clear.c b/pfsimulator/amps/mpi1/amps_clear.c
index 2eaa98f97..4b5ab52a2 100644
--- a/pfsimulator/amps/mpi1/amps_clear.c
+++ b/pfsimulator/amps/mpi1/amps_clear.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <string.h>
 #include <stdarg.h>
 
diff --git a/pfsimulator/amps/mpi1/amps_createinvoice.c b/pfsimulator/amps/mpi1/amps_createinvoice.c
index 59235a4f3..92dd3f540 100644
--- a/pfsimulator/amps/mpi1/amps_createinvoice.c
+++ b/pfsimulator/amps/mpi1/amps_createinvoice.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <string.h>
 #include <stdarg.h>
@@ -101,7 +101,7 @@ int amps_CreateInvoice(amps_Comm comm, amps_Invoice inv)
         }
         cur_pos += AMPS_CALL_BYTE_SIZEOF(comm, cur_pos, NULL, len, stride);
         break;
-	
+
       case AMPS_INVOICE_CHAR_CTYPE:
         cur_pos += AMPS_CALL_CHAR_ALIGN(comm, NULL, cur_pos, len, stride);
         if (ptr->data_type == AMPS_INVOICE_POINTER)
diff --git a/pfsimulator/amps/mpi1/amps_exchange.c b/pfsimulator/amps/mpi1/amps_exchange.c
index d274f3b4f..36c160d4e 100644
--- a/pfsimulator/amps/mpi1/amps_exchange.c
+++ b/pfsimulator/amps/mpi1/amps_exchange.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -43,8 +43,8 @@ void _amps_wait_exchange(amps_Handle handle)
                 handle->package->recv_requests, handle->package->status);
     for (i = 0; i < handle->package->num_recv; i++)
     {
-      amps_gpupacking(AMPS_UNPACK, 
-                      handle->package->recv_invoices[i], 
+      amps_gpupacking(AMPS_UNPACK,
+                      handle->package->recv_invoices[i],
                       i, &combuf, &size);
     }
     for (i = 0; i < handle->package->num_recv; i++)
@@ -59,7 +59,7 @@ void _amps_wait_exchange(amps_Handle handle)
     MPI_Datatype type = handle->package->recv_invoices[i]->mpi_type;
     if (type != MPI_DATATYPE_NULL && type != MPI_BYTE)
       MPI_Type_free(&(handle->package->recv_invoices[i]->mpi_type));
-    if(handle->package->recv_requests[i] != MPI_REQUEST_NULL)
+    if (handle->package->recv_requests[i] != MPI_REQUEST_NULL)
       MPI_Request_free(&(handle->package->recv_requests[i]));
   }
   for (i = 0; i < handle->package->num_send; i++)
@@ -67,7 +67,7 @@ void _amps_wait_exchange(amps_Handle handle)
     MPI_Datatype type = handle->package->send_invoices[i]->mpi_type;
     if (type != MPI_DATATYPE_NULL && type != MPI_BYTE)
       MPI_Type_free(&handle->package->send_invoices[i]->mpi_type);
-    if(handle->package->send_requests[i] != MPI_REQUEST_NULL)
+    if (handle->package->send_requests[i] != MPI_REQUEST_NULL)
       MPI_Request_free(&(handle->package->send_requests[i]));
   }
 }
@@ -79,11 +79,13 @@ amps_Handle amps_IExchangePackage(amps_Package package)
   int errchk;
   int i;
 
-  if(package->num_send > 0){
+  if (package->num_send > 0)
+  {
     combuf = (char**)malloc(package->num_send * sizeof(char*));
     size = (int*)malloc(package->num_send * sizeof(int));
   }
-  else{
+  else
+  {
     combuf = (char**)malloc(sizeof(char*));
     size = (int*)malloc(sizeof(int));
   }
@@ -93,12 +95,14 @@ amps_Handle amps_IExchangePackage(amps_Package package)
    *--------------------------------------------------------------------*/
   for (i = 0; i < package->num_recv; i++)
   {
-    errchk = amps_gpupacking(AMPS_GETRBUF, package->recv_invoices[i], 
-                   i, &combuf[0], &size[0]);
-    if(errchk == 0){    
+    errchk = amps_gpupacking(AMPS_GETRBUF, package->recv_invoices[i],
+                             i, &combuf[0], &size[0]);
+    if (errchk == 0)
+    {
       package->recv_invoices[i]->mpi_type = MPI_BYTE;
     }
-    else{ 
+    else
+    {
       combuf[0] = NULL;
       size[0] = 1;
       amps_create_mpi_type(amps_CommWorld, package->recv_invoices[i]);
@@ -115,12 +119,14 @@ amps_Handle amps_IExchangePackage(amps_Package package)
    *--------------------------------------------------------------------*/
   for (i = 0; i < package->num_send; i++)
   {
-    errchk = amps_gpupacking(AMPS_PACK, package->send_invoices[i], 
-                   i, &combuf[i], &size[i]);
-    if(errchk == 0){    
+    errchk = amps_gpupacking(AMPS_PACK, package->send_invoices[i],
+                             i, &combuf[i], &size[i]);
+    if (errchk == 0)
+    {
       package->send_invoices[i]->mpi_type = MPI_BYTE;
     }
-    else{
+    else
+    {
       combuf[i] = NULL;
       size[i] = 1;
       amps_create_mpi_type(amps_CommWorld, package->send_invoices[i]);
@@ -155,7 +161,7 @@ void _amps_wait_exchange(amps_Handle handle)
     {
       if (handle->package->recv_invoices[i]->mpi_type != MPI_DATATYPE_NULL)
         MPI_Type_free(&(handle->package->recv_invoices[i]->mpi_type));
-      if(handle->package->recv_requests[i] != MPI_REQUEST_NULL)
+      if (handle->package->recv_requests[i] != MPI_REQUEST_NULL)
         MPI_Request_free(&handle->package->recv_requests[i]);
     }
 
@@ -163,7 +169,7 @@ void _amps_wait_exchange(amps_Handle handle)
     {
       if (handle->package->send_invoices[i]->mpi_type != MPI_DATATYPE_NULL)
         MPI_Type_free(&handle->package->send_invoices[i]->mpi_type);
-      if(handle->package->send_requests[i] != MPI_REQUEST_NULL)
+      if (handle->package->send_requests[i] != MPI_REQUEST_NULL)
         MPI_Request_free(&handle->package->send_requests[i]);
     }
   }
diff --git a/pfsimulator/amps/mpi1/amps_finalize.c b/pfsimulator/amps/mpi1/amps_finalize.c
index 4fee01906..e6f02b2dc 100644
--- a/pfsimulator/amps/mpi1/amps_finalize.c
+++ b/pfsimulator/amps/mpi1/amps_finalize.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 
@@ -68,7 +68,7 @@ int amps_Finalize()
     MPI_Comm_free(&amps_CommNode);
     MPI_Comm_free(&amps_CommWrite);
     MPI_Comm_free(&amps_CommWorld);
-    
+
     MPI_Finalize();
   }
 #if defined(PARFLOW_HAVE_CUDA) || defined(PARFLOW_HAVE_KOKKOS)
diff --git a/pfsimulator/amps/mpi1/amps_gpupacking.cpp b/pfsimulator/amps/mpi1/amps_gpupacking.cpp
index e4892a210..5245fed66 100644
--- a/pfsimulator/amps/mpi1/amps_gpupacking.cpp
+++ b/pfsimulator/amps/mpi1/amps_gpupacking.cpp
@@ -222,7 +222,6 @@ void kokkosMemCpyDeviceToDevice(char *dest, char *src, size_t size){
   Kokkos::deep_copy(dest_view, src_view);
 }
 
-
 /**
  * @brief Device-host memcopy
  *
@@ -240,7 +239,6 @@ void kokkosMemCpyDeviceToHost(char *dest, char *src, size_t size){
 #endif
   Kokkos::deep_copy(dest_view, src_view);
 }
-
 /**
  * @brief Host-device memcopy
  *
@@ -258,7 +256,6 @@ void kokkosMemCpyHostToDevice(char *dest, char *src, size_t size){
 #endif
   Kokkos::deep_copy(dest_view, src_view);
 }
-
 /**
  * @brief Host-host memcopy
  *
@@ -350,18 +347,14 @@ void amps_gpu_free_bufs(){
 void amps_gpu_finalize(){
   amps_gpu_free_bufs();
   amps_gpu_destroy_streams();
-  if(Kokkos::is_initialized) Kokkos::finalize();
+  if(Kokkos::is_initialized() && !Kokkos::is_finalized()) Kokkos::finalize();
 }
 
 /**
  * @brief Initialize Kokkos if not initialized
  */
 void amps_kokkos_initialization(){
-  if(!Kokkos::is_initialized()){
-    Kokkos::InitArguments args;
-    args.ndevices = 1;
-    Kokkos::initialize(args);  
-  }
+  if(!Kokkos::is_initialized()) Kokkos::initialize();
 }
 
 /**
@@ -625,11 +618,10 @@ int amps_gpupacking(int action, amps_Invoice inv, int inv_num, char **buffer_out
       return __LINE__;
     }
 #endif
-
     /* Run packing or unpacking kernel */
-    using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;
+    using MDPolicyType_3D = typename Kokkos::MDRangePolicy<Kokkos::Rank<3> >;
     MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{len_x, len_y, len_z}});
-
+    
     if(action == AMPS_PACK){
       Kokkos::parallel_for(mdpolicy_3d, KOKKOS_LAMBDA(int i, int j, int k)
       {
@@ -662,11 +654,10 @@ int amps_gpupacking(int action, amps_Invoice inv, int inv_num, char **buffer_out
       });
       inv->flags &= ~AMPS_PACKED;
     }
-
     pos += size;  
     ptr = ptr->next;
   }
-
+  
   /* Check that the size is calculated right */
   // if(pos != amps_sizeof_invoice(amps_CommWorld, inv)){
     // printf("ERROR at %s:%d: The size does not match the invoice size\n", __FILE__, __LINE__);
diff --git a/pfsimulator/amps/mpi1/amps_init.c b/pfsimulator/amps/mpi1/amps_init.c
index ea8c9f8ec..9c2b616db 100644
--- a/pfsimulator/amps/mpi1/amps_init.c
+++ b/pfsimulator/amps/mpi1/amps_init.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -136,7 +136,7 @@ int amps_Init(int *argc, char **argv[])
   checkSum &= INT_MAX;
   MPI_Comm_split(amps_CommWorld, checkSum, amps_rank, &amps_CommNode);
 #endif
-  
+
   MPI_Comm_rank(amps_CommNode, &amps_node_rank);
   MPI_Comm_size(amps_CommNode, &amps_node_size);
   int color;
diff --git a/pfsimulator/amps/mpi1/amps_invoice.c b/pfsimulator/amps/mpi1/amps_invoice.c
index 872d42981..f5f4ba619 100644
--- a/pfsimulator/amps/mpi1/amps_invoice.c
+++ b/pfsimulator/amps/mpi1/amps_invoice.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <string.h>
 #include <ctype.h>
diff --git a/pfsimulator/amps/mpi1/amps_irecv.c b/pfsimulator/amps/mpi1/amps_irecv.c
index e177ace95..39ae32ff3 100644
--- a/pfsimulator/amps/mpi1/amps_irecv.c
+++ b/pfsimulator/amps/mpi1/amps_irecv.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 /*===========================================================================*/
diff --git a/pfsimulator/amps/mpi1/amps_newpackage.c b/pfsimulator/amps/mpi1/amps_newpackage.c
index 1195d991a..9fd75bbfd 100644
--- a/pfsimulator/amps/mpi1/amps_newpackage.c
+++ b/pfsimulator/amps/mpi1/amps_newpackage.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -45,12 +45,12 @@ amps_Package amps_NewPackage(amps_Comm     comm,
 
   if (num_recv + num_send)
   {
-    package->recv_requests = 
+    package->recv_requests =
       (MPI_Request*)calloc((num_recv + num_send), sizeof(MPI_Request));
 
     package->send_requests = package->recv_requests + num_recv;
 
-    package->status = 
+    package->status =
       (MPI_Status*)calloc((num_recv + num_send), sizeof(MPI_Status));
   }
 
diff --git a/pfsimulator/amps/mpi1/amps_pack.c b/pfsimulator/amps/mpi1/amps_pack.c
index 50c39dff8..7289fa675 100644
--- a/pfsimulator/amps/mpi1/amps_pack.c
+++ b/pfsimulator/amps/mpi1/amps_pack.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <string.h>
 #include <stdarg.h>
@@ -89,7 +89,7 @@ int amps_create_mpi_cont_send_type(
         MPI_Get_address(cur_pos, &mpi_displacements[element]);
         cur_pos += AMPS_CALL_BYTE_SIZEOF(comm, cur_pos, NULL, len, 1);
         break;
-	
+
       case AMPS_INVOICE_CHAR_CTYPE:
         cur_pos += AMPS_CALL_CHAR_ALIGN(comm, NULL, cur_pos, len, 1);
         MPI_Type_vector(len, 1, 1, MPI_BYTE, &mpi_types[element]);
@@ -149,13 +149,13 @@ int amps_create_mpi_cont_send_type(
 
         switch (ptr->type - AMPS_INVOICE_LAST_CTYPE)
         {
-	  case AMPS_INVOICE_BYTE_CTYPE:
+          case AMPS_INVOICE_BYTE_CTYPE:
             if (!ptr->ignore)
             {
               MPI_Type_vector(len, 1, 1, MPI_BYTE, base_type);
             }
             break;
-	    
+
           case AMPS_INVOICE_CHAR_CTYPE:
             if (!ptr->ignore)
             {
@@ -378,8 +378,8 @@ void amps_create_mpi_type(
 
         switch (ptr->type - AMPS_INVOICE_LAST_CTYPE)
         {
-	  case AMPS_INVOICE_BYTE_CTYPE:
-	    MPI_Type_vector(len, 1, stride, MPI_BYTE, base_type);
+          case AMPS_INVOICE_BYTE_CTYPE:
+            MPI_Type_vector(len, 1, stride, MPI_BYTE, base_type);
             element_size = sizeof(char);
             break;
 
diff --git a/pfsimulator/amps/mpi1/amps_recv.c b/pfsimulator/amps/mpi1/amps_recv.c
index 8de0413ae..6bb5dc0b2 100644
--- a/pfsimulator/amps/mpi1/amps_recv.c
+++ b/pfsimulator/amps/mpi1/amps_recv.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/mpi1/amps_send.c b/pfsimulator/amps/mpi1/amps_send.c
index 2218bdfee..884939bea 100644
--- a/pfsimulator/amps/mpi1/amps_send.c
+++ b/pfsimulator/amps/mpi1/amps_send.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/mpi1/amps_sizeofinvoice.c b/pfsimulator/amps/mpi1/amps_sizeofinvoice.c
index 70684c289..2146229db 100644
--- a/pfsimulator/amps/mpi1/amps_sizeofinvoice.c
+++ b/pfsimulator/amps/mpi1/amps_sizeofinvoice.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -66,7 +66,7 @@ long amps_sizeof_invoice(
         cur_pos += AMPS_CALL_BYTE_SIZEOF(comm, data, cur_pos,
                                          len, stride);
         break;
-      
+
       case AMPS_INVOICE_CHAR_CTYPE:
         cur_pos += AMPS_CALL_CHAR_ALIGN(comm, data, cur_pos, len, stride);
         cur_pos += AMPS_CALL_CHAR_SIZEOF(comm, data, cur_pos,
diff --git a/pfsimulator/amps/mpi1/amps_test.c b/pfsimulator/amps/mpi1/amps_test.c
index 0cdf2df18..0338d7df7 100644
--- a/pfsimulator/amps/mpi1/amps_test.c
+++ b/pfsimulator/amps/mpi1/amps_test.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/mpi1/amps_unpack.c b/pfsimulator/amps/mpi1/amps_unpack.c
index 25ad1065c..c2b26353f 100644
--- a/pfsimulator/amps/mpi1/amps_unpack.c
+++ b/pfsimulator/amps/mpi1/amps_unpack.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <string.h>
 #include <stdarg.h>
@@ -117,7 +117,7 @@ int amps_unpack(
         }
         break;
 
-	
+
       case AMPS_INVOICE_CHAR_CTYPE:
         if (!ptr->ignore)
         {
@@ -310,8 +310,8 @@ int amps_unpack(
 
         switch (ptr->type - AMPS_INVOICE_LAST_CTYPE)
         {
-	  case AMPS_INVOICE_BYTE_CTYPE:
-	                if (!ptr->ignore)
+          case AMPS_INVOICE_BYTE_CTYPE:
+            if (!ptr->ignore)
             {
               MPI_Type_vector(len, 1, stride, MPI_BYTE, base_type);
               element_size = sizeof(char);
diff --git a/pfsimulator/amps/mpi1/amps_vector.c b/pfsimulator/amps/mpi1/amps_vector.c
index 251cc05dd..c28ace1e8 100644
--- a/pfsimulator/amps/mpi1/amps_vector.c
+++ b/pfsimulator/amps/mpi1/amps_vector.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -100,7 +100,7 @@ void amps_vector_in(amps_Comm comm, int type, char **data, char **buf_ptr, int d
 
       switch (type)
       {
-	case AMPS_INVOICE_BYTE_CTYPE:
+        case AMPS_INVOICE_BYTE_CTYPE:
           *(char**)data += stride[dim];
           break;
 
@@ -151,7 +151,7 @@ int amps_vector_align(amps_Comm comm, int type, char **data, char **buf_ptr, int
       align = AMPS_CALL_BYTE_ALIGN(comm, NULL, *buf_ptr, len[0],
                                    stride[0]);
       break;
-    
+
     case AMPS_INVOICE_CHAR_CTYPE:
       align = AMPS_CALL_CHAR_ALIGN(comm, NULL, *buf_ptr, len[0],
                                    stride[0]);
@@ -251,7 +251,7 @@ int amps_vector_sizeof_local(amps_Comm comm, int type, char **data, char **buf_p
                                    stride[0]);
       el_size = sizeof(char);
       break;
-    
+
     case AMPS_INVOICE_CHAR_CTYPE:
       size = AMPS_CALL_CHAR_SIZEOF(comm, *buf_ptr, NULL, len[0],
                                    stride[0]);
diff --git a/pfsimulator/amps/mpi1/peekmc b/pfsimulator/amps/mpi1/peekmc
index 47f356a76..7cc6c6a84 100755
--- a/pfsimulator/amps/mpi1/peekmc
+++ b/pfsimulator/amps/mpi1/peekmc
@@ -1,7 +1,7 @@
 #! /bin/sh
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pfsimulator/amps/mpi1/run b/pfsimulator/amps/mpi1/run
index a9d8cf218..3b833be18 100755
--- a/pfsimulator/amps/mpi1/run
+++ b/pfsimulator/amps/mpi1/run
@@ -1,7 +1,7 @@
 #! /bin/sh
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pfsimulator/amps/oas3/amps.h b/pfsimulator/amps/oas3/amps.h
index a98436f51..dce290609 100644
--- a/pfsimulator/amps/oas3/amps.h
+++ b/pfsimulator/amps/oas3/amps.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #ifndef amps_include
 #define amps_include
 
@@ -270,7 +270,7 @@ typedef long amps_Clock_t;
 #define AMPS_TICKS_PER_SEC 10000
 typedef clock_t amps_CPUClock_t;
 extern long AMPS_CPU_TICKS_PER_SEC;
-#endif 
+#endif
 
 #ifdef CRAY_TIME
 typedef long amps_Clock_t;
@@ -415,160 +415,160 @@ extern amps_Buffer *amps_BufferFreeList;
 /*---------------------------------------------------------------------------*/
 /* Functions to for align                                                    */
 /*---------------------------------------------------------------------------*/
-#define AMPS_ALIGN(type, src, dest, len, stride) \
-  ((sizeof(type) -                               \
-    ((unsigned long)(dest) % sizeof(type)))      \
-   % sizeof(type));
+#define AMPS_ALIGN(type, src, dest, len, stride)       \
+        ((sizeof(type) -                               \
+          ((unsigned long)(dest) % sizeof(type)))      \
+         % sizeof(type));
 
 
 #define AMPS_CALL_BYTE_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(char, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(char, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_CHAR_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(char, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(char, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_SHORT_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(short, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(short, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_INT_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(int, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(int, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_LONG_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(long, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(long, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_FLOAT_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(float, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(float, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_DOUBLE_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(double, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(double, (_src), (_dest), (_len), (_stride))
 
 /*---------------------------------------------------------------------------*/
 /* Functions to for sizeof                                                   */
 /*---------------------------------------------------------------------------*/
 #define AMPS_SIZEOF(len, stride, size) \
-  (size_t)(len) * (size)
+        (size_t)(len) * (size)
 
-#define AMPS_CALL_BYTE_SIZEOF(_comm, _src, _dest, _len, _stride)	\
-  AMPS_SIZEOF((_len), (_stride), sizeof(char))
+#define AMPS_CALL_BYTE_SIZEOF(_comm, _src, _dest, _len, _stride)        \
+        AMPS_SIZEOF((_len), (_stride), sizeof(char))
 
 #define AMPS_CALL_CHAR_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(char))
+        AMPS_SIZEOF((_len), (_stride), sizeof(char))
 
 #define AMPS_CALL_SHORT_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(short))
+        AMPS_SIZEOF((_len), (_stride), sizeof(short))
 
 #define AMPS_CALL_INT_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(int))
+        AMPS_SIZEOF((_len), (_stride), sizeof(int))
 
 #define AMPS_CALL_LONG_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(long))
+        AMPS_SIZEOF((_len), (_stride), sizeof(long))
 
 #define AMPS_CALL_FLOAT_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(float))
+        AMPS_SIZEOF((_len), (_stride), sizeof(float))
 
 #define AMPS_CALL_DOUBLE_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(double))
+        AMPS_SIZEOF((_len), (_stride), sizeof(double))
 
 /*---------------------------------------------------------------------------*/
 /*---------------------------------------------------------------------------*/
-#define AMPS_CONVERT_OUT(type, cvt, comm, src, dest, len, stride)                                         \
-  {                                                                                                       \
-    type *ptr_src, *ptr_dest;                                                                             \
-    if ((char*)(src) != (char*)(dest))                                                                    \
-      if ((stride) == 1)                                                                                  \
-        bcopy((src), (dest), (len) * sizeof(type));                                                       \
-      else                                                                                                \
-        for (ptr_src = (type*)(src), ptr_dest = (type*)(dest); ptr_src < (type*)(src) + (len) * (stride); \
-             ptr_src += (stride), ptr_dest++)                                                             \
-          bcopy((ptr_src), (ptr_dest), sizeof(type));                                                     \
-  }
+#define AMPS_CONVERT_OUT(type, cvt, comm, src, dest, len, stride)                                               \
+        {                                                                                                       \
+          type *ptr_src, *ptr_dest;                                                                             \
+          if ((char*)(src) != (char*)(dest))                                                                    \
+          if ((stride) == 1)                                                                                    \
+          bcopy((src), (dest), (len) * sizeof(type));                                                           \
+          else                                                                                                  \
+          for (ptr_src = (type*)(src), ptr_dest = (type*)(dest); ptr_src < (type*)(src) + (len) * (stride);     \
+               ptr_src += (stride), ptr_dest++)                                                                 \
+          bcopy((ptr_src), (ptr_dest), sizeof(type));                                                           \
+        }
 
 #define AMPS_CALL_BYTE_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(char, ctohc, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(char, ctohc, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_CHAR_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(char, ctohc, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(char, ctohc, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_SHORT_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(short, ctohs, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(short, ctohs, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_INT_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(int, ctohi, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(int, ctohi, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_LONG_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(long, ctohl, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(long, ctohl, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_FLOAT_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(float, ctohf, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(float, ctohf, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_DOUBLE_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(double, ctohd, (_comm), (_src), (_dest), (_len), (_stride))
-
-
-
-#define AMPS_CONVERT_IN(type, cvt, comm, src, dest, len, stride)                         \
-  {                                                                                      \
-    char *ptr_src, *ptr_dest;                                                            \
-    if ((src) != (dest))                                                                 \
-    {                                                                                    \
-      if ((stride) == 1)                                                                 \
-      {                                                                                  \
-        bcopy((src), (dest), (size_t)(len) * sizeof(type));                              \
-      }                                                                                  \
-      else                                                                               \
-      {                                                                                  \
-        for (ptr_src = (char*)(src), (ptr_dest) = (char*)(dest);                         \
-             (ptr_dest) < (char*)(dest) + (size_t)((len) * (stride)) * sizeof(type);     \
-             (ptr_src) += sizeof(type), (ptr_dest) += sizeof(type) * (size_t)((stride))) \
-          bcopy(ptr_src, ptr_dest, sizeof(type));                                        \
-        ;                                                                                \
-      }                                                                                  \
-    }                                                                                    \
-  }
+        AMPS_CONVERT_OUT(double, ctohd, (_comm), (_src), (_dest), (_len), (_stride))
+
+
+
+#define AMPS_CONVERT_IN(type, cvt, comm, src, dest, len, stride)                               \
+        {                                                                                      \
+          char *ptr_src, *ptr_dest;                                                            \
+          if ((src) != (dest))                                                                 \
+          {                                                                                    \
+            if ((stride) == 1)                                                                 \
+            {                                                                                  \
+              bcopy((src), (dest), (size_t)(len) * sizeof(type));                              \
+            }                                                                                  \
+            else                                                                               \
+            {                                                                                  \
+              for (ptr_src = (char*)(src), (ptr_dest) = (char*)(dest);                         \
+                   (ptr_dest) < (char*)(dest) + (size_t)((len) * (stride)) * sizeof(type);     \
+                   (ptr_src) += sizeof(type), (ptr_dest) += sizeof(type) * (size_t)((stride))) \
+              bcopy(ptr_src, ptr_dest, sizeof(type));                                          \
+              ;                                                                                \
+            }                                                                                  \
+          }                                                                                    \
+        }
 
 #define AMPS_CALL_BYTE_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(char, htocc, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(char, htocc, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_CHAR_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(char, htocc, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(char, htocc, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_SHORT_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(short, htocs, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(short, htocs, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_INT_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(int, htoci, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(int, htoci, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_LONG_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(long, htocl, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(long, htocl, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_FLOAT_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(float, htocf, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(float, htocf, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_DOUBLE_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(double, htocd, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(double, htocd, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CHECK_OVERLAY(_type, _comm) 0
 
 #define AMPS_BYTE_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(char, _comm)
+        AMPS_CHECK_OVERLAY(char, _comm)
 
 #define AMPS_CHAR_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(char, _comm)
+        AMPS_CHECK_OVERLAY(char, _comm)
 
 #define AMPS_SHORT_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(short, _comm)
+        AMPS_CHECK_OVERLAY(short, _comm)
 
 #define AMPS_INT_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(int, _comm)
+        AMPS_CHECK_OVERLAY(int, _comm)
 
 #define AMPS_LONG_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(long, _comm)
+        AMPS_CHECK_OVERLAY(long, _comm)
 
 #define AMPS_FLOAT_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(float, _comm)
+        AMPS_CHECK_OVERLAY(float, _comm)
 
 #define AMPS_DOUBLE_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(double, _comm)
+        AMPS_CHECK_OVERLAY(double, _comm)
 
 /*---------------------------------------------------------------------------*/
 /* Macros for Invoice creation and deletion.                                 */
@@ -580,20 +580,20 @@ extern amps_Buffer *amps_BufferFreeList;
 /* Internal macros used to clear buffer and letter spaces.                   */
 /*---------------------------------------------------------------------------*/
 
-#define AMPS_CLEAR_INVOICE(invoice)                     \
-  {                                                     \
-    (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED; \
-    amps_ClearInvoice(invoice);                         \
-  }
-
-#define AMPS_PACK_FREE_LETTER(comm, invoice, amps_letter) \
-  if ((invoice)->combuf_flags & AMPS_INVOICE_OVERLAYED)   \
-    (invoice)->combuf_flags |= AMPS_INVOICE_ALLOCATED;    \
-  else                                                    \
-  {                                                       \
-    (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED;   \
-    amps_free((comm), (amps_letter));                     \
-  }                                                       \
+#define AMPS_CLEAR_INVOICE(invoice)                           \
+        {                                                     \
+          (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED; \
+          amps_ClearInvoice(invoice);                         \
+        }
+
+#define AMPS_PACK_FREE_LETTER(comm, invoice, amps_letter)       \
+        if ((invoice)->combuf_flags & AMPS_INVOICE_OVERLAYED)   \
+        (invoice)->combuf_flags |= AMPS_INVOICE_ALLOCATED;      \
+        else                                                    \
+        {                                                       \
+          (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED;   \
+          amps_free((comm), (amps_letter));                     \
+        }                                                       \
 
 /**
  *
@@ -897,28 +897,28 @@ extern amps_Buffer *amps_BufferFreeList;
 /*---------------------------------------------------------------------------*/
 
 #define amps_WriteChar(file, ptr, len) \
-  fwrite((ptr), sizeof(char), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_WriteShort(file, ptr, len) \
-  fwrite((ptr), sizeof(short), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(short), (len), (FILE*)(file))
 
 void amps_WriteInt(amps_File file, int *ptr, int len);
 
 #define amps_WriteLong(file, ptr, len) \
-  fwrite((ptr), sizeof(long), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(long), (len), (FILE*)(file))
 
 void amps_WriteDouble(amps_File file, double *ptr, int len);
 
 #define amps_ReadChar(file, ptr, len) \
-  fread((ptr), sizeof(char), (len), (FILE*)(file))
+        fread((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_ReadShort(file, ptr, len) \
-  fread((ptr), sizeof(short), (len), (FILE*)(file))
+        fread((ptr), sizeof(short), (len), (FILE*)(file))
 
 void amps_ReadInt(amps_File file, int *ptr, int len);
 
 #define amps_ReadLong(file, ptr, len) \
-  fread((ptr), sizeof(long), (len), (FILE*)(file))
+        fread((ptr), sizeof(long), (len), (FILE*)(file))
 
 void amps_ReadDouble(amps_File file, double *ptr, int len);
 
@@ -927,74 +927,74 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
 #ifdef AMPS_INTS_ARE_64
 
 #define amps_WriteChar(file, ptr, len) \
-  fwrite((ptr), sizeof(char), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_WriteShort(file, ptr, len) \
-  fwrite((ptr), sizeof(short), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(short), (len), (FILE*)(file))
 
 #define amps_WriteLong(file, ptr, len) \
-  fwrite((ptr), sizeof(long), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(long), (len), (FILE*)(file))
 
 #define amps_WriteFloat(file, ptr, len) \
-  fwrite((ptr), sizeof(float), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(float), (len), (FILE*)(file))
 
 #define amps_WriteDouble(file, ptr, len) \
-  fwrite((ptr), sizeof(double), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(double), (len), (FILE*)(file))
 
 
 #define amps_ReadChar(file, ptr, len) \
-  fread((ptr), sizeof(char), (len), (FILE*)(file))
+        fread((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_ReadShort(file, ptr, len) \
-  fread((ptr), sizeof(short), (len), (FILE*)(file))
+        fread((ptr), sizeof(short), (len), (FILE*)(file))
 
 #define amps_ReadLong(file, ptr, len) \
-  fread((ptr), sizeof(long), (len), (FILE*)(file))
+        fread((ptr), sizeof(long), (len), (FILE*)(file))
 
 #define amps_ReadFloat(file, ptr, len) \
-  fread((ptr), sizeof(float), (len), (FILE*)(file))
+        fread((ptr), sizeof(float), (len), (FILE*)(file))
 
 #define amps_ReadDouble(file, ptr, len) \
-  fread((ptr), sizeof(double), (len), (FILE*)(file))
+        fread((ptr), sizeof(double), (len), (FILE*)(file))
 
 #else
 
 #define amps_WriteChar(file, ptr, len) \
-  fwrite((ptr), sizeof(char), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_WriteShort(file, ptr, len) \
-  fwrite((ptr), sizeof(short), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(short), (len), (FILE*)(file))
 
 #define amps_WriteInt(file, ptr, len) \
-  fwrite((ptr), sizeof(int), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(int), (len), (FILE*)(file))
 
 #define amps_WriteLong(file, ptr, len) \
-  fwrite((ptr), sizeof(long), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(long), (len), (FILE*)(file))
 
 #define amps_WriteFloat(file, ptr, len) \
-  fwrite((ptr), sizeof(float), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(float), (len), (FILE*)(file))
 
 #define amps_WriteDouble(file, ptr, len) \
-  fwrite((ptr), sizeof(double), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(double), (len), (FILE*)(file))
 
 
 #define amps_ReadChar(file, ptr, len) \
-  fread((ptr), sizeof(char), (len), (FILE*)(file))
+        fread((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_ReadShort(file, ptr, len) \
-  fread((ptr), sizeof(short), (len), (FILE*)(file))
+        fread((ptr), sizeof(short), (len), (FILE*)(file))
 
 #define amps_ReadInt(file, ptr, len) \
-  fread((ptr), sizeof(int), (len), (FILE*)(file))
+        fread((ptr), sizeof(int), (len), (FILE*)(file))
 
 #define amps_ReadLong(file, ptr, len) \
-  fread((ptr), sizeof(long), (len), (FILE*)(file))
+        fread((ptr), sizeof(long), (len), (FILE*)(file))
 
 #define amps_ReadFloat(file, ptr, len) \
-  fread((ptr), sizeof(float), (len), (FILE*)(file))
+        fread((ptr), sizeof(float), (len), (FILE*)(file))
 
 #define amps_ReadDouble(file, ptr, len) \
-  fread((ptr), sizeof(double), (len), (FILE*)(file))
+        fread((ptr), sizeof(double), (len), (FILE*)(file))
 
 #endif
 #endif
@@ -1018,7 +1018,7 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
  * @param count Number of items of type to allocate
  * @return Pointer to the allocated dataspace
  */
-#define amps_TAlloc(type, count) ((count>0) ? (type*)malloc((unsigned int)(sizeof(type) * (count))) : NULL)
+#define amps_TAlloc(type, count) ((count > 0) ? (type*)malloc((unsigned int)(sizeof(type) * (count))) : NULL)
 
 /*===========================================================================*/
 /**
@@ -1042,7 +1042,7 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
  */
 
 #define amps_CTAlloc(type, count) \
-  ((count) ? (type*)calloc((unsigned int)(count), (unsigned int)sizeof(type)) : NULL)
+        ((count) ? (type*)calloc((unsigned int)(count), (unsigned int)sizeof(type)) : NULL)
 
 /**
  *
@@ -1064,7 +1064,7 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
 
 // SGS FIXME this should do something more than this
 #define amps_Error(name, type, comment, operation) \
-  printf("%s : %s\n", name, comment)
+        printf("%s : %s\n", name, comment)
 
 #if defined(PARFLOW_HAVE_CUDA) || defined(PARFLOW_HAVE_KOKKOS)
 /*--------------------------------------------------------------------------
@@ -1073,9 +1073,9 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
 
 /**
  * @brief Operation modes for amps_gpupacking function
- *  
+ *
  * @note See function description for amps_gpupacking.
- * 
+ *
  * @{
  */
 #define AMPS_GETRBUF 1
@@ -1096,34 +1096,38 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
 
 /**
  * @brief CUDA error handling
- * 
+ *
  * If error detected, print error message and exit.
  *
  * @param expr CUDA error (of type cudaError_t) [IN]
  */
-#define CUDA_ERRCHK( err ) (amps_cuda_error( err, __FILE__, __LINE__ ))
-static inline void amps_cuda_error(cudaError_t err, const char *file, int line) {
-	if (err != cudaSuccess) {
-		printf("\n\n%s in %s at line %d\n", cudaGetErrorString(err), file, line);
-		exit(1);
-	}
+#define CUDA_ERRCHK(err) (amps_cuda_error(err, __FILE__, __LINE__))
+static inline void amps_cuda_error(cudaError_t err, const char *file, int line)
+{
+  if (err != cudaSuccess)
+  {
+    printf("\n\n%s in %s at line %d\n", cudaGetErrorString(err), file, line);
+    exit(1);
+  }
 }
 #endif // PARFLOW_HAVE_CUDA
 
 #ifdef PARFLOW_HAVE_RMM
 /**
  * @brief RMM error handling
- * 
+ *
  * If error detected, print error message and exit.
  *
  * @param expr RMM error (of type rmmError_t) [IN]
  */
-#define RMM_ERRCHK( err ) (amps_rmm_error( err, __FILE__, __LINE__ ))
-static inline void amps_rmm_error(rmmError_t err, const char *file, int line) {
-	if (err != RMM_SUCCESS) {
-		printf("\n\n%s in %s at line %d\n", rmmGetErrorString(err), file, line);
-		exit(1);
-	}
+#define RMM_ERRCHK(err) (amps_rmm_error(err, __FILE__, __LINE__))
+static inline void amps_rmm_error(rmmError_t err, const char *file, int line)
+{
+  if (err != RMM_SUCCESS)
+  {
+    printf("\n\n%s in %s at line %d\n", rmmGetErrorString(err), file, line);
+    exit(1);
+  }
 }
 #endif // PARFLOW_HAVE_RMM
 
@@ -1153,9 +1157,9 @@ void kokkosMemSetAmps(char *ptr, size_t size);
 
 /**
  * @brief Allocates unified memory
- * 
+ *
  * If RMM library is available, pool allocation is used for better performance.
- * 
+ *
  * @note Should not be called directly.
  *
  * @param size bytes to be allocated [IN]
@@ -1163,25 +1167,25 @@ void kokkosMemSetAmps(char *ptr, size_t size);
  */
 static inline void *_amps_talloc_device(size_t size)
 {
-  void *ptr = NULL;  
-  
+  void *ptr = NULL;
+
 #ifdef PARFLOW_HAVE_RMM
-  RMM_ERRCHK(rmmAlloc(&ptr,size,0,__FILE__,__LINE__));
+  RMM_ERRCHK(rmmAlloc(&ptr, size, 0, __FILE__, __LINE__));
 #elif defined(PARFLOW_HAVE_KOKKOS)
   ptr = kokkosUVMAlloc(size);
 #elif defined(PARFLOW_HAVE_CUDA)
   CUDA_ERRCHK(cudaMallocManaged((void**)&ptr, size, cudaMemAttachGlobal));
-  // CUDA_ERRCHK(cudaHostAlloc((void**)&ptr, size, cudaHostAllocMapped));  
+  // CUDA_ERRCHK(cudaHostAlloc((void**)&ptr, size, cudaHostAllocMapped));
 #endif
-  
+
   return ptr;
 }
 
 /**
  * @brief Allocates unified memory initialized to 0
- * 
+ *
  * If RMM library is available, pool allocation is used for better performance.
- * 
+ *
  * @note Should not be called directly.
  *
  * @param size bytes to be allocated [IN]
@@ -1189,30 +1193,30 @@ static inline void *_amps_talloc_device(size_t size)
  */
 static inline void *_amps_ctalloc_device(size_t size)
 {
-  void *ptr = NULL;  
+  void *ptr = NULL;
 
 #ifdef PARFLOW_HAVE_RMM
-  RMM_ERRCHK(rmmAlloc(&ptr,size,0,__FILE__,__LINE__));
+  RMM_ERRCHK(rmmAlloc(&ptr, size, 0, __FILE__, __LINE__));
 #elif defined(PARFLOW_HAVE_KOKKOS)
   ptr = kokkosUVMAlloc(size);
 #elif defined(PARFLOW_HAVE_CUDA)
   CUDA_ERRCHK(cudaMallocManaged((void**)&ptr, size, cudaMemAttachGlobal));
   // CUDA_ERRCHK(cudaHostAlloc((void**)&ptr, size, cudaHostAllocMapped));
-#endif  
+#endif
 
 #if defined(PARFLOW_HAVE_CUDA)
-  CUDA_ERRCHK(cudaMemset(ptr, 0, size));  
+  CUDA_ERRCHK(cudaMemset(ptr, 0, size));
 #else
   // memset(ptr, 0, size);
   kokkosMemSetAmps((char*)ptr, size);
 #endif
-  
+
   return ptr;
 }
 
 /**
  * @brief Frees unified memory allocated with \ref _talloc_cuda or \ref _ctalloc_cuda
- * 
+ *
  * @note Should not be called directly.
  *
  * @param ptr a void pointer to the allocated dataspace [IN]
@@ -1220,7 +1224,7 @@ static inline void *_amps_ctalloc_device(size_t size)
 static inline void _amps_tfree_device(void *ptr)
 {
 #ifdef PARFLOW_HAVE_RMM
-  RMM_ERRCHK(rmmFree(ptr,0,__FILE__,__LINE__));
+  RMM_ERRCHK(rmmFree(ptr, 0, __FILE__, __LINE__));
 #elif defined(PARFLOW_HAVE_KOKKOS)
   kokkosUVMFree(ptr);
 #elif defined(PARFLOW_HAVE_CUDA)
@@ -1229,17 +1233,17 @@ static inline void _amps_tfree_device(void *ptr)
 #endif
 }
 
-/** 
+/**
  * Same as \ref amps_TAlloc but allocates managed memory
  */
-#define amps_TAlloc_managed(type, count) ((count>0) ? (type*)_amps_talloc_device((unsigned int)(sizeof(type) * (count))) : NULL)
+#define amps_TAlloc_managed(type, count) ((count > 0) ? (type*)_amps_talloc_device((unsigned int)(sizeof(type) * (count))) : NULL)
 
-/** 
+/**
  * Same as \ref amps_CTAlloc but allocates managed memory
  */
 #define amps_CTAlloc_managed(type, count) ((count) ? (type*)_amps_ctalloc_device((unsigned int)(sizeof(type) * (count))) : NULL)
 
-/** 
+/**
  * Same as \ref amps_TFree but deallocates managed memory
  */
 #define amps_TFree_managed(ptr) if (ptr) _amps_tfree_device(ptr); else {}
diff --git a/pfsimulator/amps/oas3/amps_allreduce.c b/pfsimulator/amps/oas3/amps_allreduce.c
index dc0ebab36..aeabc6c54 100644
--- a/pfsimulator/amps/oas3/amps_allreduce.c
+++ b/pfsimulator/amps/oas3/amps_allreduce.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -102,7 +102,7 @@ int amps_AllReduce(amps_Comm comm, amps_Invoice invoice, MPI_Op operation)
     switch (ptr->type)
     {
       case AMPS_INVOICE_BYTE_CTYPE:
-	      mpi_type = MPI_BYTE;
+        mpi_type = MPI_BYTE;
         element_size = sizeof(char);
         break;
 
@@ -148,7 +148,8 @@ int amps_AllReduce(amps_Comm comm, amps_Invoice invoice, MPI_Op operation)
     struct cudaPointerAttributes attributes;
     cudaPointerGetAttributes(&attributes, (void *)data);
 
-    if(cudaGetLastError() == cudaSuccess && attributes.type > 1){
+    if (cudaGetLastError() == cudaSuccess && attributes.type > 1)
+    {
       if (stride == 1)
         CUDA_ERRCHK(cudaMemPrefetchAsync(data, (size_t)len * element_size, cudaCpuDeviceId, 0));
       else
@@ -157,7 +158,7 @@ int amps_AllReduce(amps_Comm comm, amps_Invoice invoice, MPI_Op operation)
              ptr_src += stride * element_size)
           CUDA_ERRCHK(cudaMemPrefetchAsync(ptr_src, (size_t)element_size, cudaCpuDeviceId, 0));
 
-      CUDA_ERRCHK(cudaStreamSynchronize(0)); 
+      CUDA_ERRCHK(cudaStreamSynchronize(0));
     }
 #endif
 
diff --git a/pfsimulator/amps/oas3/amps_bcast.c b/pfsimulator/amps/oas3/amps_bcast.c
index b1d6e8698..e4341facf 100644
--- a/pfsimulator/amps/oas3/amps_bcast.c
+++ b/pfsimulator/amps/oas3/amps_bcast.c
@@ -1,38 +1,38 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
 #define vtor(node) \
-  (node > source ? node : (node) ? node - 1 : source)
+        (node > source ? node : (node) ? node - 1 : source)
 
 #define rtov(node) \
-  (node > source ? node : (node < source) ? node + 1 : 0)
+        (node > source ? node : (node < source) ? node + 1 : 0)
 
 /*===========================================================================*/
 /**
diff --git a/pfsimulator/amps/oas3/amps_clear.c b/pfsimulator/amps/oas3/amps_clear.c
index 2eaa98f97..4b5ab52a2 100644
--- a/pfsimulator/amps/oas3/amps_clear.c
+++ b/pfsimulator/amps/oas3/amps_clear.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <string.h>
 #include <stdarg.h>
 
diff --git a/pfsimulator/amps/oas3/amps_createinvoice.c b/pfsimulator/amps/oas3/amps_createinvoice.c
index 59235a4f3..92dd3f540 100644
--- a/pfsimulator/amps/oas3/amps_createinvoice.c
+++ b/pfsimulator/amps/oas3/amps_createinvoice.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <string.h>
 #include <stdarg.h>
@@ -101,7 +101,7 @@ int amps_CreateInvoice(amps_Comm comm, amps_Invoice inv)
         }
         cur_pos += AMPS_CALL_BYTE_SIZEOF(comm, cur_pos, NULL, len, stride);
         break;
-	
+
       case AMPS_INVOICE_CHAR_CTYPE:
         cur_pos += AMPS_CALL_CHAR_ALIGN(comm, NULL, cur_pos, len, stride);
         if (ptr->data_type == AMPS_INVOICE_POINTER)
diff --git a/pfsimulator/amps/oas3/amps_exchange.c b/pfsimulator/amps/oas3/amps_exchange.c
index 700daa3c9..12ad49183 100644
--- a/pfsimulator/amps/oas3/amps_exchange.c
+++ b/pfsimulator/amps/oas3/amps_exchange.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -43,8 +43,8 @@ void _amps_wait_exchange(amps_Handle handle)
                 handle->package->recv_requests, handle->package->status);
     for (i = 0; i < handle->package->num_recv; i++)
     {
-      amps_gpupacking(AMPS_UNPACK, 
-                      handle->package->recv_invoices[i], 
+      amps_gpupacking(AMPS_UNPACK,
+                      handle->package->recv_invoices[i],
                       i, &combuf, &size);
     }
     for (i = 0; i < handle->package->num_recv; i++)
@@ -59,7 +59,7 @@ void _amps_wait_exchange(amps_Handle handle)
     MPI_Datatype type = handle->package->recv_invoices[i]->mpi_type;
     if (type != MPI_DATATYPE_NULL && type != MPI_BYTE)
       MPI_Type_free(&(handle->package->recv_invoices[i]->mpi_type));
-    if(handle->package->recv_requests[i] != MPI_REQUEST_NULL)
+    if (handle->package->recv_requests[i] != MPI_REQUEST_NULL)
       MPI_Request_free(&(handle->package->recv_requests[i]));
   }
   for (i = 0; i < handle->package->num_send; i++)
@@ -67,7 +67,7 @@ void _amps_wait_exchange(amps_Handle handle)
     MPI_Datatype type = handle->package->send_invoices[i]->mpi_type;
     if (type != MPI_DATATYPE_NULL && type != MPI_BYTE)
       MPI_Type_free(&handle->package->send_invoices[i]->mpi_type);
-    if(handle->package->send_requests[i] != MPI_REQUEST_NULL)
+    if (handle->package->send_requests[i] != MPI_REQUEST_NULL)
       MPI_Request_free(&(handle->package->send_requests[i]));
   }
 }
@@ -79,11 +79,13 @@ amps_Handle amps_IExchangePackage(amps_Package package)
   int errchk;
   int i;
 
-  if(package->num_send > 0){
+  if (package->num_send > 0)
+  {
     combuf = (char**)malloc(package->num_send * sizeof(char*));
     size = (int*)malloc(package->num_send * sizeof(int));
   }
-  else{
+  else
+  {
     combuf = (char**)malloc(sizeof(char*));
     size = (int*)malloc(sizeof(int));
   }
@@ -93,12 +95,14 @@ amps_Handle amps_IExchangePackage(amps_Package package)
    *--------------------------------------------------------------------*/
   for (i = 0; i < package->num_recv; i++)
   {
-    errchk = amps_gpupacking(AMPS_GETRBUF, package->recv_invoices[i], 
-                   i, &combuf[0], &size[0]);
-    if(errchk == 0){    
+    errchk = amps_gpupacking(AMPS_GETRBUF, package->recv_invoices[i],
+                             i, &combuf[0], &size[0]);
+    if (errchk == 0)
+    {
       package->recv_invoices[i]->mpi_type = MPI_BYTE;
     }
-    else{ 
+    else
+    {
       combuf[0] = NULL;
       size[0] = 1;
       amps_create_mpi_type(oas3Comm, package->recv_invoices[i]);
@@ -115,12 +119,14 @@ amps_Handle amps_IExchangePackage(amps_Package package)
    *--------------------------------------------------------------------*/
   for (i = 0; i < package->num_send; i++)
   {
-    errchk = amps_gpupacking(AMPS_PACK, package->send_invoices[i], 
-                   i, &combuf[i], &size[i]);
-    if(errchk == 0){    
+    errchk = amps_gpupacking(AMPS_PACK, package->send_invoices[i],
+                             i, &combuf[i], &size[i]);
+    if (errchk == 0)
+    {
       package->send_invoices[i]->mpi_type = MPI_BYTE;
     }
-    else{
+    else
+    {
       combuf[i] = NULL;
       size[i] = 1;
       amps_create_mpi_type(oas3Comm, package->send_invoices[i]);
@@ -155,7 +161,7 @@ void _amps_wait_exchange(amps_Handle handle)
     {
       if (handle->package->recv_invoices[i]->mpi_type != MPI_DATATYPE_NULL)
         MPI_Type_free(&(handle->package->recv_invoices[i]->mpi_type));
-      if(handle->package->recv_requests[i] != MPI_REQUEST_NULL)
+      if (handle->package->recv_requests[i] != MPI_REQUEST_NULL)
         MPI_Request_free(&handle->package->recv_requests[i]);
     }
 
@@ -163,7 +169,7 @@ void _amps_wait_exchange(amps_Handle handle)
     {
       if (handle->package->send_invoices[i]->mpi_type != MPI_DATATYPE_NULL)
         MPI_Type_free(&handle->package->send_invoices[i]->mpi_type);
-      if(handle->package->send_requests[i] != MPI_REQUEST_NULL)
+      if (handle->package->send_requests[i] != MPI_REQUEST_NULL)
         MPI_Request_free(&handle->package->send_requests[i]);
     }
   }
diff --git a/pfsimulator/amps/oas3/amps_finalize.c b/pfsimulator/amps/oas3/amps_finalize.c
index e6b314d24..c3a11e65c 100644
--- a/pfsimulator/amps/oas3/amps_finalize.c
+++ b/pfsimulator/amps/oas3/amps_finalize.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 int dummy2_oas3 = 0;
diff --git a/pfsimulator/amps/oas3/amps_init.c b/pfsimulator/amps/oas3/amps_init.c
index 14eaa681b..351ba448a 100644
--- a/pfsimulator/amps/oas3/amps_init.c
+++ b/pfsimulator/amps/oas3/amps_init.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -123,9 +123,9 @@ int amps_Init(int *argc, char **argv[])
   CALL_oas_pfl_init(&dummy1_oas3);
   amps_mpi_initialized = TRUE;
 #ifdef __INTEL_COMPILER
-   oas3Comm = MPI_Comm_f2c(oas_pfl_vardef_mp_localcomm_); //for Intel compilers
+  oas3Comm = MPI_Comm_f2c(oas_pfl_vardef_mp_localcomm_);  //for Intel compilers
 #else
-   oas3Comm = MPI_Comm_f2c(__oas_pfl_vardef_MOD_localcomm); //for GNU compilers
+  oas3Comm = MPI_Comm_f2c(__oas_pfl_vardef_MOD_localcomm);  //for GNU compilers
 #endif
 
   MPI_Comm_size(oas3Comm, &amps_size);
@@ -209,7 +209,7 @@ int amps_Init(int *argc, char **argv[])
 /*
 #ifdef TIMING
 #ifndef CRAY_TIME
-   AMPS_CPU_TICKS_PER_SEC = sysconf(_SC_CLK_TCK);
+  AMPS_CPU_TICKS_PER_SEC = sysconf(_SC_CLK_TCK);
 #endif
 #endif
 */
diff --git a/pfsimulator/amps/oas3/amps_invoice.c b/pfsimulator/amps/oas3/amps_invoice.c
index 872d42981..f5f4ba619 100644
--- a/pfsimulator/amps/oas3/amps_invoice.c
+++ b/pfsimulator/amps/oas3/amps_invoice.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <string.h>
 #include <ctype.h>
diff --git a/pfsimulator/amps/oas3/amps_irecv.c b/pfsimulator/amps/oas3/amps_irecv.c
index e177ace95..39ae32ff3 100644
--- a/pfsimulator/amps/oas3/amps_irecv.c
+++ b/pfsimulator/amps/oas3/amps_irecv.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 /*===========================================================================*/
diff --git a/pfsimulator/amps/oas3/amps_newpackage.c b/pfsimulator/amps/oas3/amps_newpackage.c
index 1195d991a..9fd75bbfd 100644
--- a/pfsimulator/amps/oas3/amps_newpackage.c
+++ b/pfsimulator/amps/oas3/amps_newpackage.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -45,12 +45,12 @@ amps_Package amps_NewPackage(amps_Comm     comm,
 
   if (num_recv + num_send)
   {
-    package->recv_requests = 
+    package->recv_requests =
       (MPI_Request*)calloc((num_recv + num_send), sizeof(MPI_Request));
 
     package->send_requests = package->recv_requests + num_recv;
 
-    package->status = 
+    package->status =
       (MPI_Status*)calloc((num_recv + num_send), sizeof(MPI_Status));
   }
 
diff --git a/pfsimulator/amps/oas3/amps_pack.c b/pfsimulator/amps/oas3/amps_pack.c
index 50c39dff8..7289fa675 100644
--- a/pfsimulator/amps/oas3/amps_pack.c
+++ b/pfsimulator/amps/oas3/amps_pack.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <string.h>
 #include <stdarg.h>
@@ -89,7 +89,7 @@ int amps_create_mpi_cont_send_type(
         MPI_Get_address(cur_pos, &mpi_displacements[element]);
         cur_pos += AMPS_CALL_BYTE_SIZEOF(comm, cur_pos, NULL, len, 1);
         break;
-	
+
       case AMPS_INVOICE_CHAR_CTYPE:
         cur_pos += AMPS_CALL_CHAR_ALIGN(comm, NULL, cur_pos, len, 1);
         MPI_Type_vector(len, 1, 1, MPI_BYTE, &mpi_types[element]);
@@ -149,13 +149,13 @@ int amps_create_mpi_cont_send_type(
 
         switch (ptr->type - AMPS_INVOICE_LAST_CTYPE)
         {
-	  case AMPS_INVOICE_BYTE_CTYPE:
+          case AMPS_INVOICE_BYTE_CTYPE:
             if (!ptr->ignore)
             {
               MPI_Type_vector(len, 1, 1, MPI_BYTE, base_type);
             }
             break;
-	    
+
           case AMPS_INVOICE_CHAR_CTYPE:
             if (!ptr->ignore)
             {
@@ -378,8 +378,8 @@ void amps_create_mpi_type(
 
         switch (ptr->type - AMPS_INVOICE_LAST_CTYPE)
         {
-	  case AMPS_INVOICE_BYTE_CTYPE:
-	    MPI_Type_vector(len, 1, stride, MPI_BYTE, base_type);
+          case AMPS_INVOICE_BYTE_CTYPE:
+            MPI_Type_vector(len, 1, stride, MPI_BYTE, base_type);
             element_size = sizeof(char);
             break;
 
diff --git a/pfsimulator/amps/oas3/amps_print.c b/pfsimulator/amps/oas3/amps_print.c
index 2f8d902ef..a47d81d00 100644
--- a/pfsimulator/amps/oas3/amps_print.c
+++ b/pfsimulator/amps/oas3/amps_print.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <stdarg.h>
 
 #include "amps.h"
diff --git a/pfsimulator/amps/oas3/amps_recv.c b/pfsimulator/amps/oas3/amps_recv.c
index 5d15401c0..253d73ada 100644
--- a/pfsimulator/amps/oas3/amps_recv.c
+++ b/pfsimulator/amps/oas3/amps_recv.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/oas3/amps_send.c b/pfsimulator/amps/oas3/amps_send.c
index d77c5d6ca..0dec8f556 100644
--- a/pfsimulator/amps/oas3/amps_send.c
+++ b/pfsimulator/amps/oas3/amps_send.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/oas3/amps_sizeofinvoice.c b/pfsimulator/amps/oas3/amps_sizeofinvoice.c
index 70684c289..2146229db 100644
--- a/pfsimulator/amps/oas3/amps_sizeofinvoice.c
+++ b/pfsimulator/amps/oas3/amps_sizeofinvoice.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -66,7 +66,7 @@ long amps_sizeof_invoice(
         cur_pos += AMPS_CALL_BYTE_SIZEOF(comm, data, cur_pos,
                                          len, stride);
         break;
-      
+
       case AMPS_INVOICE_CHAR_CTYPE:
         cur_pos += AMPS_CALL_CHAR_ALIGN(comm, data, cur_pos, len, stride);
         cur_pos += AMPS_CALL_CHAR_SIZEOF(comm, data, cur_pos,
diff --git a/pfsimulator/amps/oas3/amps_test.c b/pfsimulator/amps/oas3/amps_test.c
index 0cdf2df18..0338d7df7 100644
--- a/pfsimulator/amps/oas3/amps_test.c
+++ b/pfsimulator/amps/oas3/amps_test.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/oas3/amps_unpack.c b/pfsimulator/amps/oas3/amps_unpack.c
index 25ad1065c..c2b26353f 100644
--- a/pfsimulator/amps/oas3/amps_unpack.c
+++ b/pfsimulator/amps/oas3/amps_unpack.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <string.h>
 #include <stdarg.h>
@@ -117,7 +117,7 @@ int amps_unpack(
         }
         break;
 
-	
+
       case AMPS_INVOICE_CHAR_CTYPE:
         if (!ptr->ignore)
         {
@@ -310,8 +310,8 @@ int amps_unpack(
 
         switch (ptr->type - AMPS_INVOICE_LAST_CTYPE)
         {
-	  case AMPS_INVOICE_BYTE_CTYPE:
-	                if (!ptr->ignore)
+          case AMPS_INVOICE_BYTE_CTYPE:
+            if (!ptr->ignore)
             {
               MPI_Type_vector(len, 1, stride, MPI_BYTE, base_type);
               element_size = sizeof(char);
diff --git a/pfsimulator/amps/oas3/amps_vector.c b/pfsimulator/amps/oas3/amps_vector.c
index 251cc05dd..c28ace1e8 100644
--- a/pfsimulator/amps/oas3/amps_vector.c
+++ b/pfsimulator/amps/oas3/amps_vector.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -100,7 +100,7 @@ void amps_vector_in(amps_Comm comm, int type, char **data, char **buf_ptr, int d
 
       switch (type)
       {
-	case AMPS_INVOICE_BYTE_CTYPE:
+        case AMPS_INVOICE_BYTE_CTYPE:
           *(char**)data += stride[dim];
           break;
 
@@ -151,7 +151,7 @@ int amps_vector_align(amps_Comm comm, int type, char **data, char **buf_ptr, int
       align = AMPS_CALL_BYTE_ALIGN(comm, NULL, *buf_ptr, len[0],
                                    stride[0]);
       break;
-    
+
     case AMPS_INVOICE_CHAR_CTYPE:
       align = AMPS_CALL_CHAR_ALIGN(comm, NULL, *buf_ptr, len[0],
                                    stride[0]);
@@ -251,7 +251,7 @@ int amps_vector_sizeof_local(amps_Comm comm, int type, char **data, char **buf_p
                                    stride[0]);
       el_size = sizeof(char);
       break;
-    
+
     case AMPS_INVOICE_CHAR_CTYPE:
       size = AMPS_CALL_CHAR_SIZEOF(comm, *buf_ptr, NULL, len[0],
                                    stride[0]);
diff --git a/pfsimulator/amps/oas3/oas3_coupler.h b/pfsimulator/amps/oas3/oas3_coupler.h
index 94113032a..59a3d59ce 100644
--- a/pfsimulator/amps/oas3/oas3_coupler.h
+++ b/pfsimulator/amps/oas3/oas3_coupler.h
@@ -1,6 +1,6 @@
 #define oas_pfl_define oas_pfl_define_
 #define CALL_oas_pfl_define(nx, ny, dx, dy, ix, iy, sw_lon, sw_lat, nlon, nlat, pfl_step, pfl_stop) \
-  oas_pfl_define(&nx, &ny, &dx, &dy, &ix, &iy, &sw_lon, &sw_lat, &nlon, &nlat, &pfl_step, &pfl_stop)
+        oas_pfl_define(&nx, &ny, &dx, &dy, &ix, &iy, &sw_lon, &sw_lat, &nlon, &nlat, &pfl_step, &pfl_stop)
 void oas_pfl_define(int *nx, int *ny,
                     double *dx, double *dy,
                     int *ix, int *iy,
@@ -10,25 +10,25 @@ void oas_pfl_define(int *nx, int *ny,
 
 #define oas_pfl_snd oas_pfl_snd_
 #define CALL_oas_pfl_snd(kid, kstep, kdata, nx, ny, kinfo, kindex) \
-  oas_pfl_snd(&kid, &kstep, &kdata, &nx, &ny, &kinfo, &kindex)
+        oas_pfl_snd(&kid, &kstep, &kdata, &nx, &ny, &kinfo, &kindex)
 void oas_pfl_snd(int *kid, int *kstep, double *kdata,
                  int *nx, int *ny, int *kinfo, int *kindex);
 
 #define oas_pfl_rcv oas_pfl_rcv_
 #define CALL_oas_pfl_rcv(kid, kstep, kdata, nx, ny, kinfo) \
-  oas_pfl_rcv(&kid, &kstep, &kdata, &nx, &ny, &kinfo)
+        oas_pfl_rcv(&kid, &kstep, &kdata, &nx, &ny, &kinfo)
 void oas_pfl_rcv(int *kid, int *kstep, double *kdata,
                  int *nx, int *ny, int *kinfo);
 
 
 #define send_fld2_clm send_fld2_clm_
 #define CALL_send_fld2_clm(pressure, saturation, topo, ix, iy, nx, ny, nz, nx_f, ny_f, pstep, porosity, dz) \
-  send_fld2_clm(pressure, saturation, topo, &ix, &iy, &nx, &ny, &nz, &nx_f, &ny_f, &pstep, porosity, dz)
+        send_fld2_clm(pressure, saturation, topo, &ix, &iy, &nx, &ny, &nz, &nx_f, &ny_f, &pstep, porosity, dz)
 void send_fld2_clm(double *pressure, double *saturation, double *topo, int *ix, int*iy,
                    int *nx, int *ny, int *nz, int *nx_f, int *ny_f, double *pstep, double *porosity, double *dz);
 
 #define receive_fld2_clm receive_fld2_clm_
 #define CALL_receive_fld2_clm(evap_trans, topo, ix, iy, nx, ny, nz, nx_f, ny_f, pstep) \
-  receive_fld2_clm(evap_trans, topo, &ix, &iy, &nx, &ny, &nz, &nx_f, &ny_f, &pstep)
+        receive_fld2_clm(evap_trans, topo, &ix, &iy, &nx, &ny, &nz, &nx_f, &ny_f, &pstep)
 void receive_fld2_clm(double *evap_trans, double *topo, int *ix, int*iy,
                       int *nx, int *ny, int *nz, int *nx_f, int *ny_f, double *pstep);
diff --git a/pfsimulator/amps/oas3/oas_pfl_define.F90 b/pfsimulator/amps/oas3/oas_pfl_define.F90
index 61bc61f72..f5d614cbb 100644
--- a/pfsimulator/amps/oas3/oas_pfl_define.F90
+++ b/pfsimulator/amps/oas3/oas_pfl_define.F90
@@ -66,7 +66,6 @@ SUBROUTINE oas_pfl_define(nx,ny,pdx,pdy,ix,iy, &
 !
 INTEGER                                    ::  part_id            ! ID returned by prism_def_partition_proto 
 INTEGER                                    ::  ii, jj, nn         ! Local Variables
-INTEGER, POINTER                           ::  mask_land(:,:)     ! Mask land
 
 REAL(KIND=8), ALLOCATABLE                  :: lglon(:,:),        &! 
                                               lglat(:,:)          ! Global Grid Centres 
@@ -84,7 +83,8 @@ SUBROUTINE oas_pfl_define(nx,ny,pdx,pdy,ix,iy, &
 REAL(KIND=8), ALLOCATABLE                  ::  clmlon(:,:),        &! 
                                                clmlat(:,:)          ! Global Grid Centres
 INTEGER                                    ::  status, pflncid,  &!
-                                               pflvarid(3)        ! CPS increased to 3,Debug netcdf output
+                                               pflvarid(3),      &! CPS increased to 3,Debug netcdf output
+                                               ib, npes
 !------------------------------------------------------------------------------
 !- End of header
 !------------------------------------------------------------------------------
@@ -93,7 +93,24 @@ SUBROUTINE oas_pfl_define(nx,ny,pdx,pdy,ix,iy, &
 !- Begin Subroutine oas_pfl_define 
 !------------------------------------------------------------------------------
 !
- !++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+! Read in land mask for each sub-domain to mask recv values from CLM
+ALLOCATE( mask_land_sub(nx,ny), stat = ierror )
+IF (ierror >0) CALL prism_abort_proto(comp_id, 'oas_pfl_define', 'Failure in allocating mask_land_sub')
+CALL MPI_Comm_size(localComm, npes, ierror)
+DO ib = 0,npes-1
+  IF (rank == ib ) THEN
+   status = nf90_open("clmgrid.nc", NF90_NOWRITE, pflncid)
+   status = nf90_inq_varid(pflncid, "LANDMASK" , pflvarid(3))
+   status = nf90_get_var(pflncid, pflvarid(3), mask_land_sub, &
+                         start = (/ix+1, iy+1/), &
+                         count = (/nx, ny/) )
+   status = nf90_close(pflncid)
+   mask_land_sub = mask_land_sub
+  ENDIF
+ENDDO
+
+
+!++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  !Global grid definition for OASIS3, written by master process for 
  !the component, i.e rank = 0 
  !+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
diff --git a/pfsimulator/amps/oas3/oas_pfl_vardef.F90 b/pfsimulator/amps/oas3/oas_pfl_vardef.F90
index 97e198c75..219b08302 100644
--- a/pfsimulator/amps/oas3/oas_pfl_vardef.F90
+++ b/pfsimulator/amps/oas3/oas_pfl_vardef.F90
@@ -49,6 +49,8 @@ MODULE oas_pfl_vardef
 INTEGER, PUBLIC                           :: OASIS_Rcv  = 1        ! return code if received field
 INTEGER, PUBLIC                           :: OASIS_idle = 0        ! return code if nothing done by oasis
 
+INTEGER, DIMENSION(:,:), ALLOCATABLE      :: mask_land             ! Mask land
+INTEGER, DIMENSION(:,:), ALLOCATABLE      :: mask_land_sub         ! Mask land
 !
 REAL(KIND=8), DIMENSION(:,:), ALLOCATABLE :: bufz                  ! Temp buffer for field transfer
 REAL(KIND=8)                              :: pfl_timestep,        &! parflow time step in hrs
diff --git a/pfsimulator/amps/oas3/peekmc b/pfsimulator/amps/oas3/peekmc
index 47f356a76..7cc6c6a84 100755
--- a/pfsimulator/amps/oas3/peekmc
+++ b/pfsimulator/amps/oas3/peekmc
@@ -1,7 +1,7 @@
 #! /bin/sh
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pfsimulator/amps/oas3/receive_fld2_clm.F90 b/pfsimulator/amps/oas3/receive_fld2_clm.F90
index 4143cfb83..9ff4b434b 100644
--- a/pfsimulator/amps/oas3/receive_fld2_clm.F90
+++ b/pfsimulator/amps/oas3/receive_fld2_clm.F90
@@ -49,7 +49,6 @@ SUBROUTINE receive_fld2_clm(evap_trans,topo,ix,iy,nx,ny,nz,nx_f,ny_f,pstep)
 
                                                                          ! All vecotrs from parflow on grid w/ ghost nodes for current proc
 !Local Variables 
-
 INTEGER                            :: i, j, k, l
 INTEGER                            :: isecs                              ! Parflow model time in seconds
 INTEGER                            :: j_incr, k_incr                     ! convert 1D vector to 3D i,j,k array
@@ -96,22 +95,20 @@ SUBROUTINE receive_fld2_clm(evap_trans,topo,ix,iy,nx,ny,nz,nx_f,ny_f,pstep)
     ENDDO
  ENDDO
 
-!
  DO k = 1, nlevsoil
    IF( trcv(k)%laction )  CALL oas_pfl_rcv( k, isecs, frcv(:,:,k),nx, ny, info )
  ENDDO
 !
- evap_trans = 0.   !CPS initialize for masking
- DO i = 1, nx
-   DO j = 1, ny
-     DO k = 1, nlevsoil 
-       IF (topo_mask(i,j) .gt. 0) THEN                    !CPS mask bug fix
-         l = 1+i + j_incr*(j) + k_incr*(topo_mask(i,j)-(k-1))  !
-         evap_trans(l) = frcv(i,j,k)
-       END IF
-     ENDDO
-   ENDDO
- ENDDO
+DO i = 1, nx
+  DO j = 1, ny
+    DO k = 1, nlevsoil 
+      IF ((topo_mask(i,j) .gt. 0) .and. (mask_land_sub(i,j) .gt. 0)) THEN                    !CPS mask bug fix
+        l = 1+i + j_incr*(j) + k_incr*(topo_mask(i,j)-(k-1))  !
+        evap_trans(l) = evap_trans(l) + frcv(i,j,k)
+      END IF
+    ENDDO
+  ENDDO
+ENDDO
 
 ! Debug ouput file
  IF ( IOASISDEBUGLVL == 1 ) THEN
diff --git a/pfsimulator/amps/oas3/run b/pfsimulator/amps/oas3/run
index a9d8cf218..3b833be18 100755
--- a/pfsimulator/amps/oas3/run
+++ b/pfsimulator/amps/oas3/run
@@ -1,7 +1,7 @@
 #! /bin/sh
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pfsimulator/amps/seq/amps.h b/pfsimulator/amps/seq/amps.h
index fe0d5ea8a..0514b8dcc 100644
--- a/pfsimulator/amps/seq/amps.h
+++ b/pfsimulator/amps/seq/amps.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef amps_include
 #define amps_include
@@ -216,13 +216,13 @@ typedef struct amps_HandleObject {
 #define PACK_HOST_TYPE 1
 #define PACK_NO_CONVERT_TYPE 2
 
-#define AMPS_ALIGN(type, dest)              \
-  ((sizeof(type) -                          \
-    ((unsigned long)(dest) % sizeof(type))) \
-   % sizeof(type));
+#define AMPS_ALIGN(type, dest)                    \
+        ((sizeof(type) -                          \
+          ((unsigned long)(dest) % sizeof(type))) \
+         % sizeof(type));
 
 #define AMPS_SIZEOF(type, len, stride) \
-  ((sizeof(type) * (len) * (stride)))
+        ((sizeof(type) * (len) * (stride)))
 
 /*---------------------------------------------------------------------------*/
 /* Macros for Invoice creation and deletion.                                 */
@@ -234,19 +234,19 @@ typedef struct amps_HandleObject {
 /* Internal macros used to clear buffer and letter spaces.                   */
 /*---------------------------------------------------------------------------*/
 #if SGS
-#define AMPS_CLEAR_INVOICE(invoice) \
-  {                                 \
-    amps_ClearInvoice(invoice);     \
-  }
-
-#define AMPS_PACK_FREE_LETTER(comm, invoice, amps_letter) \
-  if ((invoice)->combuf_flags & AMPS_INVOICE_OVERLAYED)   \
-    (invoice)->combuf_flags |= AMPS_INVOICE_ALLOCATED;    \
-  else                                                    \
-  {                                                       \
-    (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED;   \
-    pvm_freebuf(amps_letter);                             \
-  }
+#define AMPS_CLEAR_INVOICE(invoice)       \
+        {                                 \
+          amps_ClearInvoice(invoice);     \
+        }
+
+#define AMPS_PACK_FREE_LETTER(comm, invoice, amps_letter)       \
+        if ((invoice)->combuf_flags & AMPS_INVOICE_OVERLAYED)   \
+        (invoice)->combuf_flags |= AMPS_INVOICE_ALLOCATED;      \
+        else                                                    \
+        {                                                       \
+          (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED;   \
+          pvm_freebuf(amps_letter);                             \
+        }
 
 #endif
 
@@ -286,47 +286,47 @@ typedef struct amps_HandleObject {
 #define amps_SizeofDouble sizeof(double)
 
 #define amps_WriteChar(file, ptr, len) \
-  fwrite((ptr), sizeof(char), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_WriteShort(file, ptr, len) \
-  fwrite((ptr), sizeof(short), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(short), (len), (FILE*)(file))
 
 #define amps_WriteLong(file, ptr, len) \
-  fwrite((ptr), sizeof(long), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(long), (len), (FILE*)(file))
 
 #define amps_WriteFloat(file, ptr, len) \
-  fwrite((ptr), sizeof(float), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(float), (len), (FILE*)(file))
 
 #ifdef CASC_HAVE_BIGENDIAN
 
 #define amps_WriteInt(file, ptr, len) \
-  fwrite((ptr), sizeof(int), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(int), (len), (FILE*)(file))
 
 #define amps_WriteDouble(file, ptr, len) \
-  fwrite((ptr), sizeof(double), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(double), (len), (FILE*)(file))
 
 #endif
 
 
 #define amps_ReadChar(file, ptr, len) \
-  fread((ptr), sizeof(char), (len), (FILE*)(file))
+        fread((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_ReadShort(file, ptr, len) \
-  fread((ptr), sizeof(short), (len), (FILE*)(file))
+        fread((ptr), sizeof(short), (len), (FILE*)(file))
 
 #define amps_ReadLong(file, ptr, len) \
-  fread((ptr), sizeof(long), (len), (FILE*)(file))
+        fread((ptr), sizeof(long), (len), (FILE*)(file))
 
 #define amps_ReadFloat(file, ptr, len) \
-  fread((ptr), sizeof(float), (len), (FILE*)(file))
+        fread((ptr), sizeof(float), (len), (FILE*)(file))
 
 #ifdef CASC_HAVE_BIGENDIAN
 
 #define amps_ReadInt(file, ptr, len) \
-  fread((ptr), sizeof(int), (len), (FILE*)(file))
+        fread((ptr), sizeof(int), (len), (FILE*)(file))
 
 #define amps_ReadDouble(file, ptr, len) \
-  fread((ptr), sizeof(double), (len), (FILE*)(file))
+        fread((ptr), sizeof(double), (len), (FILE*)(file))
 
 #endif
 
@@ -334,24 +334,24 @@ typedef struct amps_HandleObject {
 
 #ifdef AMPS_MEMORY_ALLOC_CHECK
 
-#define amps_TAlloc(type, count)                                      \
-  {                                                                   \
-    (type*)ptr;                                                       \
-    if ((ptr = (type*)malloc((unsigned int)(sizeof(type) * (count)))) \
-        == NULL)                                                      \
-      amps_Printf("Error: out of memory in <%s> at line %d\n",        \
-                  __FILE__, __LINE__);                                \
-    ptr;                                                              \
-  }
-
-#define amps_CTAlloc(type, count)                                                         \
-  {                                                                                       \
-    (type*)ptr;                                                                           \
-    if ((ptr = (type*)calloc((unsigned int)(count), (unsigned int)sizeof(type))) == NULL) \
-      amps_Printf("Error: out of memory in <%s> at line %d\n",                            \
-                  __FILE__, __LINE__);                                                    \
-    ptr;                                                                                  \
-  }
+#define amps_TAlloc(type, count)                                            \
+        {                                                                   \
+          (type*)ptr;                                                       \
+          if ((ptr = (type*)malloc((unsigned int)(sizeof(type) * (count)))) \
+              == NULL)                                                      \
+          amps_Printf("Error: out of memory in <%s> at line %d\n",          \
+                      __FILE__, __LINE__);                                  \
+          ptr;                                                              \
+        }
+
+#define amps_CTAlloc(type, count)                                                               \
+        {                                                                                       \
+          (type*)ptr;                                                                           \
+          if ((ptr = (type*)calloc((unsigned int)(count), (unsigned int)sizeof(type))) == NULL) \
+          amps_Printf("Error: out of memory in <%s> at line %d\n",                              \
+                      __FILE__, __LINE__);                                                      \
+          ptr;                                                                                  \
+        }
 
 /* note: the `else' is required to guarantee termination of the `if' */
 #define amps_TFree(ptr) if (ptr) free(ptr); else
@@ -363,10 +363,10 @@ typedef struct amps_HandleObject {
 #else
 
 #define amps_TAlloc(type, count) \
-  ((count) ? (type*)malloc((unsigned int)(sizeof(type) * (count))) : NULL)
+        ((count) ? (type*)malloc((unsigned int)(sizeof(type) * (count))) : NULL)
 
 #define amps_CTAlloc(type, count) \
-  ((count) ? (type*)calloc((unsigned int)(count), (unsigned int)sizeof(type)) : NULL)
+        ((count) ? (type*)calloc((unsigned int)(count), (unsigned int)sizeof(type)) : NULL)
 
 /* note: the `else' is required to guarantee termination of the `if' */
 #define amps_TFree(ptr) if (ptr) free(ptr); else
diff --git a/pfsimulator/amps/seq/amps_proto.h b/pfsimulator/amps/seq/amps_proto.h
index 66f89c127..ba874921b 100644
--- a/pfsimulator/amps/seq/amps_proto.h
+++ b/pfsimulator/amps/seq/amps_proto.h
@@ -1,43 +1,43 @@
 /* amps_clear.c */
-void amps_ClearInvoice (amps_Invoice inv);
+void amps_ClearInvoice(amps_Invoice inv);
 
 /* amps_clock.c */
-void amps_clock_init (void);
-amps_Clock_t amps_Clock (void);
-amps_CPUClock_t amps_CPUClock (void);
+void amps_clock_init(void);
+amps_Clock_t amps_Clock(void);
+amps_CPUClock_t amps_CPUClock(void);
 
 /* amps_ffopen.c */
-amps_File amps_FFopen (amps_Comm comm, char *filename, char *type, long size);
+amps_File amps_FFopen(amps_Comm comm, char *filename, char *type, long size);
 
 /* amps_invoice.c */
-void amps_AppendInvoice (amps_Invoice *invoice, amps_Invoice append_invoice);
-amps_Invoice amps_new_empty_invoice (void);
-int amps_FreeInvoice (amps_Invoice inv);
-int amps_add_invoice (amps_Invoice *inv, int ignore, int type, int len_type, int len, int *ptr_len, int stride_type, int stride, int *ptr_stride, int dim_type, int dim, int *ptr_dim, int data_type, void *data);
-amps_Invoice amps_NewInvoice (const char *fmt0, ...);
-int amps_num_package_items (amps_Invoice inv);
+void amps_AppendInvoice(amps_Invoice *invoice, amps_Invoice append_invoice);
+amps_Invoice amps_new_empty_invoice(void);
+int amps_FreeInvoice(amps_Invoice inv);
+int amps_add_invoice(amps_Invoice *inv, int ignore, int type, int len_type, int len, int *ptr_len, int stride_type, int stride, int *ptr_stride, int dim_type, int dim, int *ptr_dim, int data_type, void *data);
+amps_Invoice amps_NewInvoice(const char *fmt0, ...);
+int amps_num_package_items(amps_Invoice inv);
 
 /* amps_io.c */
 void amps_ScanByte(amps_File file, char *data, int len, int stride);
-void amps_ScanChar (amps_File file, char *data, int len, int stride);
-void amps_ScanShort (amps_File file, short *data, int len, int stride);
-void amps_ScanInt (amps_File file, int *data, int len, int stride);
-void amps_ScanLong (amps_File file, long *data, int len, int stride);
-void amps_ScanFloat (amps_File file, float *data, int len, int stride);
-void amps_ScanDouble (amps_File file, double *data, int len, int stride);
-void amps_WriteDouble (amps_File file, double *ptr, int len);
-void amps_WriteInt (amps_File file, int *ptr, int len);
-void amps_ReadDouble (amps_File file, double *ptr, int len);
-void amps_ReadInt (amps_File file, int *ptr, int len);
-void amps_WriteInt (amps_File file, int *ptr, int len);
-void amps_ReadInt (amps_File file, int *ptr, int len);
+void amps_ScanChar(amps_File file, char *data, int len, int stride);
+void amps_ScanShort(amps_File file, short *data, int len, int stride);
+void amps_ScanInt(amps_File file, int *data, int len, int stride);
+void amps_ScanLong(amps_File file, long *data, int len, int stride);
+void amps_ScanFloat(amps_File file, float *data, int len, int stride);
+void amps_ScanDouble(amps_File file, double *data, int len, int stride);
+void amps_WriteDouble(amps_File file, double *ptr, int len);
+void amps_WriteInt(amps_File file, int *ptr, int len);
+void amps_ReadDouble(amps_File file, double *ptr, int len);
+void amps_ReadInt(amps_File file, int *ptr, int len);
+void amps_WriteInt(amps_File file, int *ptr, int len);
+void amps_ReadInt(amps_File file, int *ptr, int len);
 
 /* amps_newpackage.c */
-amps_Package amps_NewPackage (amps_Comm comm, int num_send, int *dest, amps_Invoice *send_invoices, int num_recv, int *src, amps_Invoice *recv_invoices);
-void amps_FreePackage (amps_Package package);
+amps_Package amps_NewPackage(amps_Comm comm, int num_send, int *dest, amps_Invoice *send_invoices, int num_recv, int *src, amps_Invoice *recv_invoices);
+void amps_FreePackage(amps_Package package);
 
 /* amps_sfbcast.c */
-int amps_SFBCast (amps_Comm comm, amps_File file, amps_Invoice invoice);
+int amps_SFBCast(amps_Comm comm, amps_File file, amps_Invoice invoice);
 
 void _amps_Abort(
                  char *message,
diff --git a/pfsimulator/amps/seq/peekmc b/pfsimulator/amps/seq/peekmc
index 1c13301c1..142c89efa 100755
--- a/pfsimulator/amps/seq/peekmc
+++ b/pfsimulator/amps/seq/peekmc
@@ -1,7 +1,7 @@
 #! /bin/sh
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pfsimulator/amps/seq/run b/pfsimulator/amps/seq/run
index efdd709ec..f9dae1f03 100755
--- a/pfsimulator/amps/seq/run
+++ b/pfsimulator/amps/seq/run
@@ -1,7 +1,7 @@
 #!/bin/sh
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pfsimulator/amps/smpi/amps.h b/pfsimulator/amps/smpi/amps.h
index 063147602..38ec55a5c 100644
--- a/pfsimulator/amps/smpi/amps.h
+++ b/pfsimulator/amps/smpi/amps.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #ifndef amps_include
 #define amps_include
 
@@ -400,154 +400,154 @@ extern amps_Buffer *amps_BufferFreeList;
 /*---------------------------------------------------------------------------*/
 /* Functions to for align                                                    */
 /*---------------------------------------------------------------------------*/
-#define AMPS_ALIGN(type, src, dest, len, stride) \
-  ((sizeof(type) -                               \
-    ((unsigned long)(dest) % sizeof(type)))      \
-   % sizeof(type));
+#define AMPS_ALIGN(type, src, dest, len, stride)       \
+        ((sizeof(type) -                               \
+          ((unsigned long)(dest) % sizeof(type)))      \
+         % sizeof(type));
 
 #define AMPS_CALL_BYTE_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(char, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(char, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_CHAR_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(char, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(char, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_SHORT_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(short, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(short, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_INT_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(int, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(int, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_LONG_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(long, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(long, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_FLOAT_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(float, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(float, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_DOUBLE_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(double, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(double, (_src), (_dest), (_len), (_stride))
 
 /*---------------------------------------------------------------------------*/
 /* Functions to for sizeof                                                   */
 /*---------------------------------------------------------------------------*/
 #define AMPS_SIZEOF(len, stride, size) \
-  (len) * (size)
+        (len) * (size)
 
-#define AMPS_CALL_BYTE_SIZEOF(_comm, _src, _dest, _len, _stride)	\
-  AMPS_SIZEOF((_len), (_stride), sizeof(char))
+#define AMPS_CALL_BYTE_SIZEOF(_comm, _src, _dest, _len, _stride)        \
+        AMPS_SIZEOF((_len), (_stride), sizeof(char))
 
 #define AMPS_CALL_CHAR_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(char))
+        AMPS_SIZEOF((_len), (_stride), sizeof(char))
 
 #define AMPS_CALL_SHORT_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(short))
+        AMPS_SIZEOF((_len), (_stride), sizeof(short))
 
 #define AMPS_CALL_INT_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(int))
+        AMPS_SIZEOF((_len), (_stride), sizeof(int))
 
 #define AMPS_CALL_LONG_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(long))
+        AMPS_SIZEOF((_len), (_stride), sizeof(long))
 
 #define AMPS_CALL_FLOAT_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(float))
+        AMPS_SIZEOF((_len), (_stride), sizeof(float))
 
 #define AMPS_CALL_DOUBLE_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(double))
+        AMPS_SIZEOF((_len), (_stride), sizeof(double))
 
 /*---------------------------------------------------------------------------*/
 /*---------------------------------------------------------------------------*/
-#define AMPS_CONVERT_OUT(type, cvt, comm, src, dest, len, stride)                                         \
-  {                                                                                                       \
-    type *ptr_src, *ptr_dest;                                                                             \
-    if ((char*)(src) != (char*)(dest))                                                                    \
-    {									                                  \
-      if ((stride) == 1)                                                                                  \
-        bcopy((src), (dest), (len) * sizeof(type));                                                       \
-      else                                                                                                \
-        for (ptr_src = (type*)(src), ptr_dest = (type*)(dest); ptr_src < (type*)(src) + (len) * (stride); \
-             ptr_src += (stride), ptr_dest++)                                                             \
-          bcopy((ptr_src), (ptr_dest), sizeof(type));                                                     \
-    }									                                  \
-  }
+#define AMPS_CONVERT_OUT(type, cvt, comm, src, dest, len, stride)                                               \
+        {                                                                                                       \
+          type *ptr_src, *ptr_dest;                                                                             \
+          if ((char*)(src) != (char*)(dest))                                                                    \
+          {                                                                                                     \
+            if ((stride) == 1)                                                                                  \
+            bcopy((src), (dest), (len) * sizeof(type));                                                         \
+            else                                                                                                \
+            for (ptr_src = (type*)(src), ptr_dest = (type*)(dest); ptr_src < (type*)(src) + (len) * (stride);   \
+                 ptr_src += (stride), ptr_dest++)                                                               \
+            bcopy((ptr_src), (ptr_dest), sizeof(type));                                                         \
+          }                                                                                                     \
+        }
 
 #define AMPS_CALL_BYTE_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(char, ctohc, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(char, ctohc, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_CHAR_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(char, ctohc, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(char, ctohc, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_SHORT_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(short, ctohs, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(short, ctohs, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_INT_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(int, ctohi, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(int, ctohi, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_LONG_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(long, ctohl, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(long, ctohl, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_FLOAT_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(float, ctohf, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(float, ctohf, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_DOUBLE_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(double, ctohd, (_comm), (_src), (_dest), (_len), (_stride))
-
-#define AMPS_CONVERT_IN(type, cvt, comm, src, dest, len, stride)               \
-  {                                                                            \
-    char *ptr_src, *ptr_dest;                                                  \
-    if ((src) != (dest))                                                       \
-    {									       \
-      if ((stride) == 1)                                                       \
-        bcopy((src), (dest), (len) * sizeof(type));                            \
-      else                                                                     \
-        for (ptr_src = (char*)(src), (ptr_dest) = (char*)(dest);               \
-             (ptr_dest) < (char*)(dest) + (len) * (stride) * sizeof(type);     \
-             (ptr_src) += sizeof(type), (ptr_dest) += sizeof(type) * (stride)) \
-          bcopy((ptr_src), (ptr_dest), sizeof(type));                          \
-    }									       \
-  }
+        AMPS_CONVERT_OUT(double, ctohd, (_comm), (_src), (_dest), (_len), (_stride))
+
+#define AMPS_CONVERT_IN(type, cvt, comm, src, dest, len, stride)                     \
+        {                                                                            \
+          char *ptr_src, *ptr_dest;                                                  \
+          if ((src) != (dest))                                                       \
+          {                                                                          \
+            if ((stride) == 1)                                                       \
+            bcopy((src), (dest), (len) * sizeof(type));                              \
+            else                                                                     \
+            for (ptr_src = (char*)(src), (ptr_dest) = (char*)(dest);                 \
+                 (ptr_dest) < (char*)(dest) + (len) * (stride) * sizeof(type);       \
+                 (ptr_src) += sizeof(type), (ptr_dest) += sizeof(type) * (stride))   \
+            bcopy((ptr_src), (ptr_dest), sizeof(type));                              \
+          }                                                                          \
+        }
 
 #define AMPS_CALL_BYTE_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(char, htocc, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(char, htocc, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_CHAR_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(char, htocc, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(char, htocc, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_SHORT_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(short, htocs, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(short, htocs, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_INT_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(int, htoci, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(int, htoci, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_LONG_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(long, htocl, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(long, htocl, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_FLOAT_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(float, htocf, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(float, htocf, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_DOUBLE_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(double, htocd, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(double, htocd, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CHECK_OVERLAY(_type, _comm) 0
 
 #define AMPS_BYTE_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(char, _comm)
+        AMPS_CHECK_OVERLAY(char, _comm)
 
 #define AMPS_CHAR_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(char, _comm)
+        AMPS_CHECK_OVERLAY(char, _comm)
 
 #define AMPS_SHORT_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(short, _comm)
+        AMPS_CHECK_OVERLAY(short, _comm)
 
 #define AMPS_INT_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(int, _comm)
+        AMPS_CHECK_OVERLAY(int, _comm)
 
 #define AMPS_LONG_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(long, _comm)
+        AMPS_CHECK_OVERLAY(long, _comm)
 
 #define AMPS_FLOAT_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(float, _comm)
+        AMPS_CHECK_OVERLAY(float, _comm)
 
 #define AMPS_DOUBLE_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(double, _comm)
+        AMPS_CHECK_OVERLAY(double, _comm)
 
 /*---------------------------------------------------------------------------*/
 /* Macros for Invoice creation and deletion.                                 */
@@ -559,20 +559,20 @@ extern amps_Buffer *amps_BufferFreeList;
 /* Internal macros used to clear buffer and letter spaces.                   */
 /*---------------------------------------------------------------------------*/
 
-#define AMPS_CLEAR_INVOICE(invoice)                     \
-  {                                                     \
-    (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED; \
-    amps_ClearInvoice(invoice);                         \
-  }
-
-#define AMPS_PACK_FREE_LETTER(comm, invoice, amps_letter) \
-  if ((invoice)->combuf_flags & AMPS_INVOICE_OVERLAYED)   \
-    (invoice)->combuf_flags |= AMPS_INVOICE_ALLOCATED;    \
-  else                                                    \
-  {                                                       \
-    (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED;   \
-    amps_free((comm), (amps_letter));                     \
-  }                                                       \
+#define AMPS_CLEAR_INVOICE(invoice)                           \
+        {                                                     \
+          (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED; \
+          amps_ClearInvoice(invoice);                         \
+        }
+
+#define AMPS_PACK_FREE_LETTER(comm, invoice, amps_letter)       \
+        if ((invoice)->combuf_flags & AMPS_INVOICE_OVERLAYED)   \
+        (invoice)->combuf_flags |= AMPS_INVOICE_ALLOCATED;      \
+        else                                                    \
+        {                                                       \
+          (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED;   \
+          amps_free((comm), (amps_letter));                     \
+        }                                                       \
 
 /**
  *      amps_Printf("Freeing letter: %x\n", amps_letter); \
@@ -877,35 +877,35 @@ extern amps_Buffer *amps_BufferFreeList;
 /* On the nodes store numbers with wrong endian so we need to swap           */
 /*---------------------------------------------------------------------------*/
 
-#define amps_WriteByte(file, ptr, len)			\
-  fwrite((ptr), sizeof(char), (len), (FILE*)(file))
+#define amps_WriteByte(file, ptr, len)                  \
+        fwrite((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_WriteChar(file, ptr, len) \
-  fwrite((ptr), sizeof(char), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_WriteShort(file, ptr, len) \
-  fwrite((ptr), sizeof(short), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(short), (len), (FILE*)(file))
 
 void amps_WriteInt(amps_File file, int *ptr, int len);
 
 #define amps_WriteLong(file, ptr, len) \
-  fwrite((ptr), sizeof(long), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(long), (len), (FILE*)(file))
 
 void amps_WriteDouble(amps_File file, double *ptr, int len);
 
 #define amps_ReadWrite(file, ptr, len) \
-  fread((ptr), sizeof(char), (len), (FILE*)(file))
+        fread((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_ReadChar(file, ptr, len) \
-  fread((ptr), sizeof(char), (len), (FILE*)(file))
+        fread((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_ReadShort(file, ptr, len) \
-  fread((ptr), sizeof(short), (len), (FILE*)(file))
+        fread((ptr), sizeof(short), (len), (FILE*)(file))
 
 void amps_ReadInt(amps_File file, int *ptr, int len);
 
 #define amps_ReadLong(file, ptr, len) \
-  fread((ptr), sizeof(long), (len), (FILE*)(file))
+        fread((ptr), sizeof(long), (len), (FILE*)(file))
 
 void amps_ReadDouble(amps_File file, double *ptr, int len);
 
@@ -914,86 +914,86 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
 #ifdef AMPS_INTS_ARE_64
 
 #define amps_WriteByte(file, ptr, len) \
-  fwrite((ptr), sizeof(char), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_WriteChar(file, ptr, len) \
-  fwrite((ptr), sizeof(char), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_WriteShort(file, ptr, len) \
-  fwrite((ptr), sizeof(short), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(short), (len), (FILE*)(file))
 
 #define amps_WriteLong(file, ptr, len) \
-  fwrite((ptr), sizeof(long), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(long), (len), (FILE*)(file))
 
 #define amps_WriteFloat(file, ptr, len) \
-  fwrite((ptr), sizeof(float), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(float), (len), (FILE*)(file))
 
 #define amps_WriteDouble(file, ptr, len) \
-  fwrite((ptr), sizeof(double), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(double), (len), (FILE*)(file))
 
 
 #define amps_ReadByte(file, ptr, len) \
-  fread((ptr), sizeof(char), (len), (FILE*)(file))
+        fread((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_ReadChar(file, ptr, len) \
-  fread((ptr), sizeof(char), (len), (FILE*)(file))
+        fread((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_ReadShort(file, ptr, len) \
-  fread((ptr), sizeof(short), (len), (FILE*)(file))
+        fread((ptr), sizeof(short), (len), (FILE*)(file))
 
 #define amps_ReadLong(file, ptr, len) \
-  fread((ptr), sizeof(long), (len), (FILE*)(file))
+        fread((ptr), sizeof(long), (len), (FILE*)(file))
 
 #define amps_ReadFloat(file, ptr, len) \
-  fread((ptr), sizeof(float), (len), (FILE*)(file))
+        fread((ptr), sizeof(float), (len), (FILE*)(file))
 
 #define amps_ReadDouble(file, ptr, len) \
-  fread((ptr), sizeof(double), (len), (FILE*)(file))
+        fread((ptr), sizeof(double), (len), (FILE*)(file))
 
 #else
 
 #define amps_WriteByte(file, ptr, len) \
-  fwrite((ptr), sizeof(char), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_WriteChar(file, ptr, len) \
-  fwrite((ptr), sizeof(char), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_WriteShort(file, ptr, len) \
-  fwrite((ptr), sizeof(short), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(short), (len), (FILE*)(file))
 
 #define amps_WriteInt(file, ptr, len) \
-  fwrite((ptr), sizeof(int), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(int), (len), (FILE*)(file))
 
 #define amps_WriteLong(file, ptr, len) \
-  fwrite((ptr), sizeof(long), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(long), (len), (FILE*)(file))
 
 #define amps_WriteFloat(file, ptr, len) \
-  fwrite((ptr), sizeof(float), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(float), (len), (FILE*)(file))
 
 #define amps_WriteDouble(file, ptr, len) \
-  fwrite((ptr), sizeof(double), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(double), (len), (FILE*)(file))
 
 
 #define amps_ReadByte(file, ptr, len) \
-  fread((ptr), sizeof(char), (len), (FILE*)(file))
+        fread((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_ReadChar(file, ptr, len) \
-  fread((ptr), sizeof(char), (len), (FILE*)(file))
+        fread((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_ReadShort(file, ptr, len) \
-  fread((ptr), sizeof(short), (len), (FILE*)(file))
+        fread((ptr), sizeof(short), (len), (FILE*)(file))
 
 #define amps_ReadInt(file, ptr, len) \
-  fread((ptr), sizeof(int), (len), (FILE*)(file))
+        fread((ptr), sizeof(int), (len), (FILE*)(file))
 
 #define amps_ReadLong(file, ptr, len) \
-  fread((ptr), sizeof(long), (len), (FILE*)(file))
+        fread((ptr), sizeof(long), (len), (FILE*)(file))
 
 #define amps_ReadFloat(file, ptr, len) \
-  fread((ptr), sizeof(float), (len), (FILE*)(file))
+        fread((ptr), sizeof(float), (len), (FILE*)(file))
 
 #define amps_ReadDouble(file, ptr, len) \
-  fread((ptr), sizeof(double), (len), (FILE*)(file))
+        fread((ptr), sizeof(double), (len), (FILE*)(file))
 
 #endif
 #endif
@@ -1017,7 +1017,7 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
  * @param count Number of items of type to allocate
  * @return Pointer to the allocated dataspace
  */
-#define amps_TAlloc(type, count) ((count>0) ? (type*)malloc((unsigned int)(sizeof(type) * (count))) : NULL)
+#define amps_TAlloc(type, count) ((count > 0) ? (type*)malloc((unsigned int)(sizeof(type) * (count))) : NULL)
 
 /*===========================================================================*/
 /**
@@ -1041,7 +1041,7 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
  */
 
 #define amps_CTAlloc(type, count) \
-  ((count) ? (type*)calloc((unsigned int)(count), (unsigned int)sizeof(type)) : NULL)
+        ((count) ? (type*)calloc((unsigned int)(count), (unsigned int)sizeof(type)) : NULL)
 
 /**
  *
@@ -1064,7 +1064,7 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
 
 // SGS FIXME this should do something more than this
 #define amps_Error(name, type, comment, operation) \
-  printf("%s : %s\n", name, comment)
+        printf("%s : %s\n", name, comment)
 
 #ifdef PARFLOW_HAVE_CUDA
 /*--------------------------------------------------------------------------
@@ -1073,33 +1073,37 @@ void amps_ReadDouble(amps_File file, double *ptr, int len);
 
 /**
  * @brief CUDA error handling.
- * 
+ *
  * If error detected, print error message and exit.
  *
  * @param expr CUDA error (of type cudaError_t) [IN]
  */
-#define CUDA_ERRCHK( err ) (amps_cuda_error( err, __FILE__, __LINE__ ))
-static inline void amps_cuda_error(cudaError_t err, const char *file, int line) {
-	if (err != cudaSuccess) {
-		printf("\n\n%s in %s at line %d\n", cudaGetErrorString(err), file, line);
-		exit(1);
-	}
+#define CUDA_ERRCHK(err) (amps_cuda_error(err, __FILE__, __LINE__))
+static inline void amps_cuda_error(cudaError_t err, const char *file, int line)
+{
+  if (err != cudaSuccess)
+  {
+    printf("\n\n%s in %s at line %d\n", cudaGetErrorString(err), file, line);
+    exit(1);
+  }
 }
 
 #ifdef PARFLOW_HAVE_RMM
 /**
  * @brief RMM error handling.
- * 
+ *
  * If error detected, print error message and exit.
  *
  * @param expr RMM error (of type rmmError_t) [IN]
  */
-#define RMM_ERRCHK( err ) (amps_rmm_error( err, __FILE__, __LINE__ ))
-static inline void amps_rmm_error(rmmError_t err, const char *file, int line) {
-	if (err != RMM_SUCCESS) {
-		printf("\n\n%s in %s at line %d\n", rmmGetErrorString(err), file, line);
-		exit(1);
-	}
+#define RMM_ERRCHK(err) (amps_rmm_error(err, __FILE__, __LINE__))
+static inline void amps_rmm_error(rmmError_t err, const char *file, int line)
+{
+  if (err != RMM_SUCCESS)
+  {
+    printf("\n\n%s in %s at line %d\n", rmmGetErrorString(err), file, line);
+    exit(1);
+  }
 }
 #endif
 
@@ -1109,9 +1113,9 @@ static inline void amps_rmm_error(rmmError_t err, const char *file, int line) {
 
 /**
  * @brief Allocates unified memory.
- * 
+ *
  * If RMM library is available, pool allocation is used for better performance.
- * 
+ *
  * @note Should not be called directly.
  *
  * @param size bytes to be allocated [IN]
@@ -1119,23 +1123,23 @@ static inline void amps_rmm_error(rmmError_t err, const char *file, int line) {
  */
 static inline void *_amps_talloc_cuda(size_t size)
 {
-  void *ptr = NULL;  
-  
+  void *ptr = NULL;
+
 #ifdef PARFLOW_HAVE_RMM
-  RMM_ERRCHK(rmmAlloc(&ptr,size,0,__FILE__,__LINE__));
+  RMM_ERRCHK(rmmAlloc(&ptr, size, 0, __FILE__, __LINE__));
 #else
   CUDA_ERRCHK(cudaMallocManaged((void**)&ptr, size, cudaMemAttachGlobal));
-  // CUDA_ERRCHK(cudaHostAlloc((void**)&ptr, size, cudaHostAllocMapped));  
+  // CUDA_ERRCHK(cudaHostAlloc((void**)&ptr, size, cudaHostAllocMapped));
 #endif
-  
+
   return ptr;
 }
 
 /**
  * @brief Allocates unified memory initialized to 0.
- * 
+ *
  * If RMM library is available, pool allocation is used for better performance.
- * 
+ *
  * @note Should not be called directly.
  *
  * @param size bytes to be allocated [IN]
@@ -1143,23 +1147,23 @@ static inline void *_amps_talloc_cuda(size_t size)
  */
 static inline void *_amps_ctalloc_cuda(size_t size)
 {
-  void *ptr = NULL;  
+  void *ptr = NULL;
 
 #ifdef PARFLOW_HAVE_RMM
-  RMM_ERRCHK(rmmAlloc(&ptr,size,0,__FILE__,__LINE__));
+  RMM_ERRCHK(rmmAlloc(&ptr, size, 0, __FILE__, __LINE__));
 #else
   CUDA_ERRCHK(cudaMallocManaged((void**)&ptr, size, cudaMemAttachGlobal));
   // CUDA_ERRCHK(cudaHostAlloc((void**)&ptr, size, cudaHostAllocMapped));
-#endif  
+#endif
   // memset(ptr, 0, size);
-  CUDA_ERRCHK(cudaMemset(ptr, 0, size));  
-  
+  CUDA_ERRCHK(cudaMemset(ptr, 0, size));
+
   return ptr;
 }
 
 /**
  * @brief Frees unified memory allocated with \ref _talloc_cuda or \ref _ctalloc_cuda.
- * 
+ *
  * @note Should not be called directly.
  *
  * @param ptr a void pointer to the allocated dataspace [IN]
@@ -1167,25 +1171,25 @@ static inline void *_amps_ctalloc_cuda(size_t size)
 static inline void _amps_tfree_cuda(void *ptr)
 {
 #ifdef PARFLOW_HAVE_RMM
-  RMM_ERRCHK(rmmFree(ptr,0,__FILE__,__LINE__));
+  RMM_ERRCHK(rmmFree(ptr, 0, __FILE__, __LINE__));
 #else
   CUDA_ERRCHK(cudaFree(ptr));
   // CUDA_ERRCHK(cudaFreeHost(ptr));
 #endif
 }
 
-/** 
- * Same as \ref amps_TAlloc but allocates managed memory (CUDA required) 
+/**
+ * Same as \ref amps_TAlloc but allocates managed memory (CUDA required)
  */
-#define amps_TAlloc_managed(type, count) ((count>0) ? (type*)_amps_talloc_cuda((unsigned int)(sizeof(type) * (count))) : NULL)
+#define amps_TAlloc_managed(type, count) ((count > 0) ? (type*)_amps_talloc_cuda((unsigned int)(sizeof(type) * (count))) : NULL)
 
-/** 
- * Same as \ref amps_CTAlloc but allocates managed memory 
+/**
+ * Same as \ref amps_CTAlloc but allocates managed memory
  */
 #define amps_CTAlloc_managed(type, count) ((count) ? (type*)_amps_ctalloc_cuda((unsigned int)(sizeof(type) * (count))) : NULL)
 
-/** 
- * Same as \ref amps_TFree but deallocates managed memory 
+/**
+ * Same as \ref amps_TFree but deallocates managed memory
  */
 #define amps_TFree_managed(ptr) if (ptr) _amps_tfree_cuda(ptr); else {}
 
diff --git a/pfsimulator/amps/smpi/amps_allreduce.c b/pfsimulator/amps/smpi/amps_allreduce.c
index f04ea3ca7..82777e256 100644
--- a/pfsimulator/amps/smpi/amps_allreduce.c
+++ b/pfsimulator/amps/smpi/amps_allreduce.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 #include <string.h>
@@ -66,7 +66,7 @@ int operation;
         case amps_Max:
           switch (ptr->type)
           {
-	    case AMPS_INVOICE_BYTE_CTYPE:
+            case AMPS_INVOICE_BYTE_CTYPE:
               pos_dest += AMPS_CALL_BYTE_ALIGN(comm, NULL,
                                                pos_dest, len, 1);
               pos_src += AMPS_CALL_BYTE_ALIGN(comm, NULL,
@@ -162,7 +162,7 @@ int operation;
         case amps_Min:
           switch (ptr->type)
           {
-	    case AMPS_INVOICE_BYTE_CTYPE:
+            case AMPS_INVOICE_BYTE_CTYPE:
               pos_dest += AMPS_CALL_BYTE_ALIGN(comm, NULL,
                                                pos_dest, len, 1);
               pos_src += AMPS_CALL_BYTE_ALIGN(comm, NULL,
@@ -258,7 +258,7 @@ int operation;
         case amps_Add:
           switch (ptr->type)
           {
-	    case AMPS_INVOICE_BYTE_CTYPE:
+            case AMPS_INVOICE_BYTE_CTYPE:
               pos_dest += AMPS_CALL_BYTE_ALIGN(comm, NULL,
                                                pos_dest, len, 1);
               pos_src += AMPS_CALL_BYTE_ALIGN(comm, NULL,
@@ -342,13 +342,14 @@ int operation;
                    pos_dest += sizeof(double), pos_src += sizeof(double))
                 *(double*)pos_dest += *(double*)pos_src;
               break;
-	    default:
-	    {
-	      amps_Error("amps_pack", INVALID_INVOICE, "Invalid invoice type", HALT);
-	      pos_dest = 0;
-	      pos_src = 0;
-	    }
-	    break;
+
+            default:
+            {
+              amps_Error("amps_pack", INVALID_INVOICE, "Invalid invoice type", HALT);
+              pos_dest = 0;
+              pos_src = 0;
+            }
+            break;
           }
           break;
       }
diff --git a/pfsimulator/amps/smpi/amps_bcast.c b/pfsimulator/amps/smpi/amps_bcast.c
index 698e5a676..1e90d46af 100644
--- a/pfsimulator/amps/smpi/amps_bcast.c
+++ b/pfsimulator/amps/smpi/amps_bcast.c
@@ -1,38 +1,38 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
 #define vtor(node) \
-  (node > source ? node : (node) ? node - 1 : source)
+        (node > source ? node : (node) ? node - 1 : source)
 
 #define rtov(node) \
-  (node > source ? node : (node < source) ? node + 1 : 0)
+        (node > source ? node : (node < source) ? node + 1 : 0)
 
 /*===========================================================================*/
 /**
diff --git a/pfsimulator/amps/smpi/amps_clear.c b/pfsimulator/amps/smpi/amps_clear.c
index 913cb46d0..fb4a5062a 100644
--- a/pfsimulator/amps/smpi/amps_clear.c
+++ b/pfsimulator/amps/smpi/amps_clear.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <string.h>
 #include <stdarg.h>
 
diff --git a/pfsimulator/amps/smpi/amps_createinvoice.c b/pfsimulator/amps/smpi/amps_createinvoice.c
index 05df72fba..e39543bfd 100644
--- a/pfsimulator/amps/smpi/amps_createinvoice.c
+++ b/pfsimulator/amps/smpi/amps_createinvoice.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <string.h>
 #include <stdarg.h>
@@ -86,91 +86,91 @@ int amps_CreateInvoice(amps_Comm comm, amps_Invoice inv)
       case AMPS_INVOICE_BYTE_CTYPE:
         cur_pos += AMPS_CALL_BYTE_ALIGN(comm, NULL, cur_pos, len, stride);
         if (ptr->data_type == AMPS_INVOICE_POINTER)
-	{
+        {
           if (stride == 1)
             *((void**)(ptr->data)) = cur_pos;
           else
             *((void**)(ptr->data)) =
               malloc(sizeof(char) * len * stride);
-	}
+        }
         cur_pos += AMPS_CALL_BYTE_SIZEOF(comm, cur_pos, NULL, len, stride);
         break;
 
       case AMPS_INVOICE_CHAR_CTYPE:
         cur_pos += AMPS_CALL_CHAR_ALIGN(comm, NULL, cur_pos, len, stride);
         if (ptr->data_type == AMPS_INVOICE_POINTER)
-	{
+        {
           if (stride == 1)
             *((void**)(ptr->data)) = cur_pos;
           else
             *((void**)(ptr->data)) =
               malloc(sizeof(char) * len * stride);
-	}
+        }
         cur_pos += AMPS_CALL_CHAR_SIZEOF(comm, cur_pos, NULL, len, stride);
         break;
 
       case AMPS_INVOICE_SHORT_CTYPE:
         cur_pos += AMPS_CALL_SHORT_ALIGN(comm, NULL, cur_pos, len, stride);
         if (ptr->data_type == AMPS_INVOICE_POINTER)
-	{
+        {
           if (stride == 1)
             *((void**)(ptr->data)) = cur_pos;
           else
             *((void**)(ptr->data)) =
               malloc(sizeof(short) * len * stride);
-	}
+        }
         cur_pos += AMPS_CALL_SHORT_SIZEOF(comm, cur_pos, NULL, len, stride);
         break;
 
       case AMPS_INVOICE_INT_CTYPE:
         cur_pos += AMPS_CALL_INT_ALIGN(comm, NULL, cur_pos, len, stride);
         if (ptr->data_type == AMPS_INVOICE_POINTER)
-	{
+        {
           if (stride == 1)
             *((void**)(ptr->data)) = cur_pos;
           else
             *((void**)(ptr->data)) =
               malloc(sizeof(int) * len * stride);
-	}
+        }
         cur_pos += AMPS_CALL_INT_SIZEOF(comm, cur_pos, NULL, len, stride);
         break;
 
       case AMPS_INVOICE_LONG_CTYPE:
         cur_pos += AMPS_CALL_LONG_ALIGN(comm, NULL, cur_pos, len, stride);
         if (ptr->data_type == AMPS_INVOICE_POINTER)
-	{
+        {
           if (stride == 1)
             *((void**)(ptr->data)) = cur_pos;
           else
             *((void**)(ptr->data)) =
               malloc(sizeof(long) * len * stride);
-	}
+        }
         cur_pos += AMPS_CALL_LONG_SIZEOF(comm, cur_pos, NULL, len, stride);
         break;
 
       case AMPS_INVOICE_FLOAT_CTYPE:
         cur_pos += AMPS_CALL_FLOAT_ALIGN(comm, NULL, cur_pos, len, stride);
         if (ptr->data_type == AMPS_INVOICE_POINTER)
-	{
+        {
           if (stride == 1)
             *((void**)(ptr->data)) = cur_pos;
           else
             *((void**)(ptr->data)) =
               malloc(sizeof(float) * len * stride);
-	}
+        }
         cur_pos += AMPS_CALL_FLOAT_SIZEOF(comm, cur_pos, NULL, len, stride);
         break;
 
       case AMPS_INVOICE_DOUBLE_CTYPE:
         cur_pos += AMPS_CALL_DOUBLE_ALIGN(comm, NULL, cur_pos, len, stride);
         if (ptr->data_type == AMPS_INVOICE_POINTER)
-	{
+        {
           if (stride == 1)
             *((void**)(ptr->data)) = cur_pos;
           else
             *((void**)(ptr->data)) =
               malloc(sizeof(double) * len * stride);
-	}
+        }
         cur_pos += AMPS_CALL_DOUBLE_SIZEOF(comm, cur_pos, NULL, len, stride);
         break;
     }
diff --git a/pfsimulator/amps/smpi/amps_exchange.c b/pfsimulator/amps/smpi/amps_exchange.c
index 6073a29b5..bd9e2a603 100644
--- a/pfsimulator/amps/smpi/amps_exchange.c
+++ b/pfsimulator/amps/smpi/amps_exchange.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <sys/times.h>
 #include "amps.h"
@@ -221,15 +221,15 @@ void _amps_wait_exchange(amps_Handle handle)
  *         AMPS_CLEAR_INVOICE(handle -> package -> recv_invoices[i]);
  */
       }
-    } 
+    }
     for (i = 0; i < handle->package->num_recv; i++)
     {
-      if(handle->package->recv_requests[i] != MPI_REQUEST_NULL)
+      if (handle->package->recv_requests[i] != MPI_REQUEST_NULL)
         MPI_Request_free(&(handle->package->recv_requests[i]));
     }
     for (i = 0; i < handle->package->num_send; i++)
     {
-      if(handle->package->send_requests[i] != MPI_REQUEST_NULL)
+      if (handle->package->send_requests[i] != MPI_REQUEST_NULL)
         MPI_Request_free(&(handle->package->send_requests[i]));
     }
   }
diff --git a/pfsimulator/amps/smpi/amps_finalize.c b/pfsimulator/amps/smpi/amps_finalize.c
index e28ab9586..4dad72c5a 100644
--- a/pfsimulator/amps/smpi/amps_finalize.c
+++ b/pfsimulator/amps/smpi/amps_finalize.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 
diff --git a/pfsimulator/amps/smpi/amps_init.c b/pfsimulator/amps/smpi/amps_init.c
index 5ddb1b5d3..232847fc1 100644
--- a/pfsimulator/amps/smpi/amps_init.c
+++ b/pfsimulator/amps/smpi/amps_init.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -137,7 +137,7 @@ int amps_Init(int *argc, char **argv[])
   checkSum &= INT_MAX;
   MPI_Comm_split(amps_CommWorld, checkSum, amps_rank, &amps_CommNode);
 #endif
-  
+
   MPI_Comm_rank(amps_CommNode, &amps_node_rank);
   MPI_Comm_size(amps_CommNode, &amps_node_size);
   int color;
diff --git a/pfsimulator/amps/smpi/amps_invoice.c b/pfsimulator/amps/smpi/amps_invoice.c
index 606a314d9..0fdd014d8 100644
--- a/pfsimulator/amps/smpi/amps_invoice.c
+++ b/pfsimulator/amps/smpi/amps_invoice.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <string.h>
 #include <ctype.h>
diff --git a/pfsimulator/amps/smpi/amps_irecv.c b/pfsimulator/amps/smpi/amps_irecv.c
index e177ace95..39ae32ff3 100644
--- a/pfsimulator/amps/smpi/amps_irecv.c
+++ b/pfsimulator/amps/smpi/amps_irecv.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 /*===========================================================================*/
diff --git a/pfsimulator/amps/smpi/amps_newpackage.c b/pfsimulator/amps/smpi/amps_newpackage.c
index 3f2509b9a..d2a12c865 100644
--- a/pfsimulator/amps/smpi/amps_newpackage.c
+++ b/pfsimulator/amps/smpi/amps_newpackage.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/smpi/amps_pack.c b/pfsimulator/amps/smpi/amps_pack.c
index 6b76f323f..5ba5c9e71 100644
--- a/pfsimulator/amps/smpi/amps_pack.c
+++ b/pfsimulator/amps/smpi/amps_pack.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -300,7 +300,7 @@ amps_Invoice inv;
 
         switch (ptr->type - AMPS_INVOICE_LAST_CTYPE)
         {
-	  case AMPS_INVOICE_BYTE_CTYPE:
+          case AMPS_INVOICE_BYTE_CTYPE:
             MPI_Type_vector(len, 1, stride, MPI_BYTE, base_type);
             element_size = sizeof(char);
             break;
@@ -334,9 +334,10 @@ amps_Invoice inv;
             MPI_Type_vector(len, 1, stride, MPI_DOUBLE, base_type);
             element_size = sizeof(double);
             break;
-	  default:
-	    amps_Error("amps_pack", INVALID_INVOICE, "Invalid invoice type", HALT);
-	    element_size = INT_MIN;
+
+          default:
+            amps_Error("amps_pack", INVALID_INVOICE, "Invalid invoice type", HALT);
+            element_size = INT_MIN;
         }
 
         base_size = element_size * (len + (len - 1) * (stride - 1));
@@ -346,9 +347,9 @@ amps_Invoice inv;
           if (i == dim - 1)
           {
             MPI_Type_create_hvector(ptr->ptr_len[i], 1,
-                             base_size +
-                             (ptr->ptr_stride[i] - 1) * element_size,
-                             *base_type, &mpi_types[element]);
+                                    base_size +
+                                    (ptr->ptr_stride[i] - 1) * element_size,
+                                    *base_type, &mpi_types[element]);
             base_size = base_size * ptr->ptr_len[i]
                         + (ptr->ptr_stride[i] - 1) * (ptr->ptr_len[i] - 1)
                         * element_size;
@@ -357,9 +358,9 @@ amps_Invoice inv;
           else
           {
             MPI_Type_create_hvector(ptr->ptr_len[i], 1,
-                             base_size +
-                             (ptr->ptr_stride[i] - 1) * element_size,
-                             *base_type, new_type);
+                                    base_size +
+                                    (ptr->ptr_stride[i] - 1) * element_size,
+                                    *base_type, new_type);
             base_size = base_size * ptr->ptr_len[i]
                         + (ptr->ptr_stride[i] - 1) * (ptr->ptr_len[i] - 1)
                         * element_size;
@@ -388,10 +389,10 @@ amps_Invoice inv;
   }
 
   MPI_Type_create_struct(inv->num,
-                  mpi_block_len,
-                  mpi_displacements,
-                  mpi_types,
-                  &inv->mpi_type);
+                         mpi_block_len,
+                         mpi_displacements,
+                         mpi_types,
+                         &inv->mpi_type);
 
   for (element = 0; element < inv->num; element++)
   {
diff --git a/pfsimulator/amps/smpi/amps_print.c b/pfsimulator/amps/smpi/amps_print.c
index 2f8d902ef..a47d81d00 100644
--- a/pfsimulator/amps/smpi/amps_print.c
+++ b/pfsimulator/amps/smpi/amps_print.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <stdarg.h>
 
 #include "amps.h"
diff --git a/pfsimulator/amps/smpi/amps_proto.h b/pfsimulator/amps/smpi/amps_proto.h
index 143ae5069..4be1febec 100644
--- a/pfsimulator/amps/smpi/amps_proto.h
+++ b/pfsimulator/amps/smpi/amps_proto.h
@@ -1,134 +1,134 @@
 /* amps_allreduce.c */
-int amps_AllReduce (amps_Comm comm, amps_Invoice invoice, int operation);
+int amps_AllReduce(amps_Comm comm, amps_Invoice invoice, int operation);
 
 /* amps_bcast.c */
-int amps_BCast (amps_Comm comm, int source, amps_Invoice invoice);
+int amps_BCast(amps_Comm comm, int source, amps_Invoice invoice);
 
 /* amps_clear.c */
-void amps_ClearInvoice (amps_Invoice inv);
+void amps_ClearInvoice(amps_Invoice inv);
 
 /* amps_clock.c */
-void amps_clock_init (void);
-amps_Clock_t amps_Clock (void);
-amps_CPUClock_t amps_CPUClock (void);
+void amps_clock_init(void);
+amps_Clock_t amps_Clock(void);
+amps_CPUClock_t amps_CPUClock(void);
 
 /* amps_createinvoice.c */
-int amps_CreateInvoice (amps_Comm comm, amps_Invoice inv);
+int amps_CreateInvoice(amps_Comm comm, amps_Invoice inv);
 
 /* amps_exchange.c */
-int _amps_send_sizes (amps_Package package, int **sizes);
-int _amps_recv_sizes (amps_Package package);
-void _amps_wait_exchange (amps_Handle handle);
-amps_Handle amps_IExchangePackage (amps_Package package);
+int _amps_send_sizes(amps_Package package, int **sizes);
+int _amps_recv_sizes(amps_Package package);
+void _amps_wait_exchange(amps_Handle handle);
+amps_Handle amps_IExchangePackage(amps_Package package);
 
 /* amps_ffopen.c */
-amps_File amps_FFopen (amps_Comm comm, char *filename, char *type, long size);
+amps_File amps_FFopen(amps_Comm comm, char *filename, char *type, long size);
 
 /* amps_finalize.c */
-int amps_Finalize (void);
+int amps_Finalize(void);
 
 /* amps_find_powers.c */
-void amps_FindPowers (int N, int *log, int *Nnext, int *Nprev);
+void amps_FindPowers(int N, int *log, int *Nnext, int *Nprev);
 
 /* amps_fopen.c */
-amps_File amps_Fopen (char *filename, char *type);
+amps_File amps_Fopen(char *filename, char *type);
 
 /* amps_init.c */
-int MAIN__ (void);
-int amps_Init (int *argc, char **argv []);
+int MAIN__(void);
+int amps_Init(int *argc, char **argv []);
 int amps_EmbeddedInit(void);
 int amps_EmbeddedInitComm(MPI_Comm com);
 
 /* amps_invoice.c */
-void amps_AppendInvoice (amps_Invoice *invoice, amps_Invoice append_invoice);
-amps_Invoice amps_new_empty_invoice (void);
-int amps_FreeInvoice (amps_Invoice inv);
-int amps_add_invoice (amps_Invoice *inv, int ignore, int type, int len_type, int len, int *ptr_len, int stride_type, int stride, int *ptr_stride, int dim_type, int dim, int *ptr_dim, int data_type, void *data);
-amps_Invoice amps_NewInvoice (const char *fmt0, ...);
-int amps_num_package_items (amps_Invoice inv);
+void amps_AppendInvoice(amps_Invoice *invoice, amps_Invoice append_invoice);
+amps_Invoice amps_new_empty_invoice(void);
+int amps_FreeInvoice(amps_Invoice inv);
+int amps_add_invoice(amps_Invoice *inv, int ignore, int type, int len_type, int len, int *ptr_len, int stride_type, int stride, int *ptr_stride, int dim_type, int dim, int *ptr_dim, int data_type, void *data);
+amps_Invoice amps_NewInvoice(const char *fmt0, ...);
+int amps_num_package_items(amps_Invoice inv);
 
 /* amps_io.c */
-void amps_ScanByte (amps_File file, char *data, int len, int stride);
-void amps_ScanChar (amps_File file, char *data, int len, int stride);
-void amps_ScanShort (amps_File file, short *data, int len, int stride);
-void amps_ScanInt (amps_File file, int *data, int len, int stride);
-void amps_ScanLong (amps_File file, long *data, int len, int stride);
-void amps_ScanFloat (amps_File file, float *data, int len, int stride);
-void amps_ScanDouble (amps_File file, double *data, int len, int stride);
-void amps_WriteDouble (amps_File file, double *ptr, int len);
-void amps_WriteInt (amps_File file, int *ptr, int len);
-void amps_ReadDouble (amps_File file, double *ptr, int len);
-void amps_ReadInt (amps_File file, int *ptr, int len);
-void amps_WriteInt (amps_File file, int *ptr, int len);
-void amps_ReadInt (amps_File file, int *ptr, int len);
+void amps_ScanByte(amps_File file, char *data, int len, int stride);
+void amps_ScanChar(amps_File file, char *data, int len, int stride);
+void amps_ScanShort(amps_File file, short *data, int len, int stride);
+void amps_ScanInt(amps_File file, int *data, int len, int stride);
+void amps_ScanLong(amps_File file, long *data, int len, int stride);
+void amps_ScanFloat(amps_File file, float *data, int len, int stride);
+void amps_ScanDouble(amps_File file, double *data, int len, int stride);
+void amps_WriteDouble(amps_File file, double *ptr, int len);
+void amps_WriteInt(amps_File file, int *ptr, int len);
+void amps_ReadDouble(amps_File file, double *ptr, int len);
+void amps_ReadInt(amps_File file, int *ptr, int len);
+void amps_WriteInt(amps_File file, int *ptr, int len);
+void amps_ReadInt(amps_File file, int *ptr, int len);
 
 /* amps_irecv.c */
-amps_Handle amps_IRecv (amps_Comm comm, int source, amps_Invoice invoice);
+amps_Handle amps_IRecv(amps_Comm comm, int source, amps_Invoice invoice);
 
 /* amps_newhandle.c */
-amps_Handle amps_NewHandle (amps_Comm comm, int id, amps_Invoice invoice, amps_Package package);
+amps_Handle amps_NewHandle(amps_Comm comm, int id, amps_Invoice invoice, amps_Package package);
 void amps_FreeHandle(amps_Handle handle);
 
 /* amps_newpackage.c */
-amps_Package amps_NewPackage (amps_Comm comm, int num_send, int *dest, amps_Invoice *send_invoices, int num_recv, int *src, amps_Invoice *recv_invoices);
-void amps_FreePackage (amps_Package package);
-amps_Package amps_NewPackage (amps_Comm comm, int num_send, int *dest, amps_Invoice *send_invoices, int num_recv, int *src, amps_Invoice *recv_invoices);
-void amps_FreePackage (amps_Package package);
+amps_Package amps_NewPackage(amps_Comm comm, int num_send, int *dest, amps_Invoice *send_invoices, int num_recv, int *src, amps_Invoice *recv_invoices);
+void amps_FreePackage(amps_Package package);
+amps_Package amps_NewPackage(amps_Comm comm, int num_send, int *dest, amps_Invoice *send_invoices, int num_recv, int *src, amps_Invoice *recv_invoices);
+void amps_FreePackage(amps_Package package);
 
 /* amps_pack.c */
-void amps_create_mpi_type (amps_Comm comm, amps_Invoice inv);
-int amps_pack (amps_Comm comm, amps_Invoice inv, char **buffer);
+void amps_create_mpi_type(amps_Comm comm, amps_Invoice inv);
+int amps_pack(amps_Comm comm, amps_Invoice inv, char **buffer);
 
 /* amps_print.c */
 FILE* amps_SetConsole(FILE* stream);
-void amps_Printf (const char *fmt, ...);
+void amps_Printf(const char *fmt, ...);
 
 /* amps_recv.c */
-char *amps_recvb (int src, int *size);
-int amps_Recv (amps_Comm comm, int source, amps_Invoice invoice);
+char *amps_recvb(int src, int *size);
+int amps_Recv(amps_Comm comm, int source, amps_Invoice invoice);
 
 /* amps_send.c */
-int amps_xsend (amps_Comm comm, int dest, char *buffer, int size);
-int amps_Send (amps_Comm comm, int dest, amps_Invoice invoice);
+int amps_xsend(amps_Comm comm, int dest, char *buffer, int size);
+int amps_Send(amps_Comm comm, int dest, amps_Invoice invoice);
 
 /* amps_sfbcast.c */
-int amps_SFBCast (amps_Comm comm, amps_File file, amps_Invoice invoice);
+int amps_SFBCast(amps_Comm comm, amps_File file, amps_Invoice invoice);
 
 /* amps_sfclose.c */
-int amps_SFclose (amps_File file);
+int amps_SFclose(amps_File file);
 
 /* amps_sfopen.c */
-amps_File amps_SFopen (const char *filename, const char *type);
+amps_File amps_SFopen(const char *filename, const char *type);
 
 /* amps_sizeofinvoice.c */
-long amps_sizeof_invoice (amps_Comm comm, amps_Invoice inv);
+long amps_sizeof_invoice(amps_Comm comm, amps_Invoice inv);
 
 /* amps_test.c */
-int amps_Test (amps_Handle handle);
+int amps_Test(amps_Handle handle);
 
 /* amps_unpack.c */
-int amps_unpack (amps_Comm comm, amps_Invoice inv, char *buffer);
+int amps_unpack(amps_Comm comm, amps_Invoice inv, char *buffer);
 
 /* amps_vector.c */
-void amps_vector_in (amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
-void amps_vector_out (amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
-int amps_vector_align (amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
-int amps_vector_sizeof_buffer (amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
-int amps_vector_sizeof_local (amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
+void amps_vector_in(amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
+void amps_vector_out(amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
+int amps_vector_align(amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
+int amps_vector_sizeof_buffer(amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
+int amps_vector_sizeof_local(amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
 
 /* amps_wait.c */
-int amps_Wait (amps_Handle handle);
+int amps_Wait(amps_Handle handle);
 
 /* signal.c */
-void handler_ill (void);
-void handler_bus (void);
-void handler_seg (void);
-void handler_sys (void);
-void handler_fpe (void);
-void handler_division (void);
-void handler_overflow (void);
-void handler_invalid (void);
-void Fsignal (void);
+void handler_ill(void);
+void handler_bus(void);
+void handler_seg(void);
+void handler_sys(void);
+void handler_fpe(void);
+void handler_division(void);
+void handler_overflow(void);
+void handler_invalid(void);
+void Fsignal(void);
 
 void _amps_Abort(
                  char *message,
diff --git a/pfsimulator/amps/smpi/amps_recv.c b/pfsimulator/amps/smpi/amps_recv.c
index 72fcb9805..56bd07907 100644
--- a/pfsimulator/amps/smpi/amps_recv.c
+++ b/pfsimulator/amps/smpi/amps_recv.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/smpi/amps_send.c b/pfsimulator/amps/smpi/amps_send.c
index e8e092163..d4bcfb881 100644
--- a/pfsimulator/amps/smpi/amps_send.c
+++ b/pfsimulator/amps/smpi/amps_send.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/smpi/amps_sizeofinvoice.c b/pfsimulator/amps/smpi/amps_sizeofinvoice.c
index 4d65c32eb..4488ab3d0 100644
--- a/pfsimulator/amps/smpi/amps_sizeofinvoice.c
+++ b/pfsimulator/amps/smpi/amps_sizeofinvoice.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/smpi/amps_test.c b/pfsimulator/amps/smpi/amps_test.c
index 0cdf2df18..0338d7df7 100644
--- a/pfsimulator/amps/smpi/amps_test.c
+++ b/pfsimulator/amps/smpi/amps_test.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
diff --git a/pfsimulator/amps/smpi/amps_unpack.c b/pfsimulator/amps/smpi/amps_unpack.c
index 4c53050b4..5c63f6756 100644
--- a/pfsimulator/amps/smpi/amps_unpack.c
+++ b/pfsimulator/amps/smpi/amps_unpack.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <string.h>
 #include <stdarg.h>
 
diff --git a/pfsimulator/amps/smpi/amps_vector.c b/pfsimulator/amps/smpi/amps_vector.c
index bdc9e7209..7f746f005 100644
--- a/pfsimulator/amps/smpi/amps_vector.c
+++ b/pfsimulator/amps/smpi/amps_vector.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 
@@ -107,7 +107,7 @@ int *stride;
 
       switch (type)
       {
-	case AMPS_INVOICE_BYTE_CTYPE:
+        case AMPS_INVOICE_BYTE_CTYPE:
           *(char**)data += stride[dim];
           break;
 
@@ -224,7 +224,7 @@ int *stride;
 
       switch (type)
       {
-	case AMPS_INVOICE_BYTE_CTYPE:
+        case AMPS_INVOICE_BYTE_CTYPE:
           *(char**)data += stride[dim];
           break;
 
@@ -304,6 +304,7 @@ int *stride;
       align = AMPS_CALL_DOUBLE_ALIGN(comm, NULL, *buf_ptr, len[0],
                                      stride[0]);
       break;
+
     default:
       amps_Error("amps_pack", INVALID_INVOICE, "Invalid invoice type", HALT);
       align = INT_MIN;
@@ -354,6 +355,7 @@ int *stride;
     case AMPS_INVOICE_DOUBLE_CTYPE:
       size = AMPS_CALL_DOUBLE_SIZEOF(comm, *buf_ptr, NULL, len[0], 1);
       break;
+
     default:
       amps_Error("amps_pack", INVALID_INVOICE, "Invalid invoice type", HALT);
       size = INT_MIN;
diff --git a/pfsimulator/amps/smpi/peekmc b/pfsimulator/amps/smpi/peekmc
index 826f88cb5..cd73231a4 100644
--- a/pfsimulator/amps/smpi/peekmc
+++ b/pfsimulator/amps/smpi/peekmc
@@ -1,7 +1,7 @@
 #! /bin/sh
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pfsimulator/amps/smpi/run b/pfsimulator/amps/smpi/run
index a9d8cf218..3b833be18 100644
--- a/pfsimulator/amps/smpi/run
+++ b/pfsimulator/amps/smpi/run
@@ -1,7 +1,7 @@
 #! /bin/sh
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pfsimulator/amps/test/src/amps_test.h b/pfsimulator/amps/test/src/amps_test.h
index 05b1ab874..e4d48fa2c 100644
--- a/pfsimulator/amps/test/src/amps_test.h
+++ b/pfsimulator/amps/test/src/amps_test.h
@@ -1,44 +1,44 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #ifndef AMPS_TEST_H
 #define AMPS_TEST_H
 
 int amps_check_result(int result)
 {
-  if(result)
+  if (result)
   {
     printf("FAILED\n");
   }
   else
   {
     int me = amps_Rank(amps_CommWorld);
-    
-    if(me == 0)
+
+    if (me == 0)
     {
       printf("PASSED\n");
     }
@@ -49,28 +49,27 @@ int amps_check_result(int result)
 
 int amps_compare_files(char *filename1, char *filename2)
 {
-
   FILE *file1 = fopen(filename1, "r");
   FILE *file2 = fopen(filename2, "r");
 
   char ch1;
   char ch2;
-  
+
   do
   {
     ch1 = fgetc(file1);
     ch2 = fgetc(file2);
-        
+
     if (ch1 != ch2)
     {
       return 1;
     }
-
-  } while (ch1 != EOF && ch2 != EOF);
+  }
+  while (ch1 != EOF && ch2 != EOF);
 
   fclose(file1);
   fclose(file2);
-  
+
   if (ch1 == EOF && ch2 == EOF)
   {
     return 0;
diff --git a/pfsimulator/amps/test/src/ampsring.c b/pfsimulator/amps/test/src/ampsring.c
index 463afd547..193978e8c 100644
--- a/pfsimulator/amps/test/src/ampsring.c
+++ b/pfsimulator/amps/test/src/ampsring.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 double          ring_sync();
 
 #include <stdio.h>
@@ -82,7 +82,7 @@ char *argv[];
   amps_Finalize();
 }
 
-double          ring_sync(reps, len, left, right)
+double ring_sync(reps, len, left, right)
 int reps, len, left, right;
 {
   amps_Invoice invoice;
diff --git a/pfsimulator/amps/test/src/irixshmem.c b/pfsimulator/amps/test/src/irixshmem.c
index 7220e71fb..ea43c41f9 100644
--- a/pfsimulator/amps/test/src/irixshmem.c
+++ b/pfsimulator/amps/test/src/irixshmem.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <amps.h>
 
 #define method "Shared Memory Memcpy"
diff --git a/pfsimulator/amps/test/src/runt b/pfsimulator/amps/test/src/runt
index 2bb0c02f3..30988d105 100755
--- a/pfsimulator/amps/test/src/runt
+++ b/pfsimulator/amps/test/src/runt
@@ -1,7 +1,7 @@
 #! /bin/sh
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pfsimulator/amps/test/src/runtests b/pfsimulator/amps/test/src/runtests
index c5f76d9d4..ce11c0f72 100755
--- a/pfsimulator/amps/test/src/runtests
+++ b/pfsimulator/amps/test/src/runtests
@@ -1,7 +1,7 @@
 #!/bin/sh
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pfsimulator/amps/test/src/shmemput.c b/pfsimulator/amps/test/src/shmemput.c
index 4c2418ab2..379f7958c 100644
--- a/pfsimulator/amps/test/src/shmemput.c
+++ b/pfsimulator/amps/test/src/shmemput.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <mpp/shmem.h>
 #include <math.h>
 
diff --git a/pfsimulator/amps/test/src/test1.c b/pfsimulator/amps/test/src/test1.c
index 1f343673d..473e1f18e 100644
--- a/pfsimulator/amps/test/src/test1.c
+++ b/pfsimulator/amps/test/src/test1.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*
  * This test prints out information size and rank information on all
  * the nodes and then amps_Exits
diff --git a/pfsimulator/amps/test/src/test10.c b/pfsimulator/amps/test/src/test10.c
index 1f8ba402a..4b884bef3 100644
--- a/pfsimulator/amps/test/src/test10.c
+++ b/pfsimulator/amps/test/src/test10.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /* The following needs to be in the input file test9.input */
 /*
  * 11
@@ -81,61 +81,63 @@ int ReadAndCheckFile(char* filename, int loop)
       result |= 1;
     }
 
-    if(amps_Fscanf(file, "%d ", &string_length) != 1)
+    if (amps_Fscanf(file, "%d ", &string_length) != 1)
     {
       amps_Printf("ERROR: reading int\n");
       result |= 1;
-    };
-    
-    if(amps_Fscanf(file, "%s", recvd_string) != 1)
+    }
+    ;
+
+    if (amps_Fscanf(file, "%s", recvd_string) != 1)
     {
       amps_Printf("ERROR: reading string\n");
       result |= 1;
-    };
+    }
+    ;
 
     for (i = 0; i < shorts_length; i++)
     {
-      if(amps_Fscanf(file, "%hd ", &recvd_shorts[i]) != 1)
+      if (amps_Fscanf(file, "%hd ", &recvd_shorts[i]) != 1)
       {
-	amps_Printf("ERROR: reading short\n");
-	result |= 1;
+        amps_Printf("ERROR: reading short\n");
+        result |= 1;
       }
     }
 
     for (i = 0; i < ints_length; i++)
     {
-      if(amps_Fscanf(file, "%d ", &recvd_ints[i]) != 1)
+      if (amps_Fscanf(file, "%d ", &recvd_ints[i]) != 1)
       {
-	amps_Printf("ERROR: reading short\n");
-	result |= 1;
+        amps_Printf("ERROR: reading short\n");
+        result |= 1;
       }
     }
 
     for (i = 0; i < longs_length; i++)
     {
-      if(amps_Fscanf(file, "%ld ", &recvd_longs[i]) != 1)
+      if (amps_Fscanf(file, "%ld ", &recvd_longs[i]) != 1)
       {
-	amps_Printf("ERROR: reading short\n");
-	result |= 1;
+        amps_Printf("ERROR: reading short\n");
+        result |= 1;
       }
     }
 
     for (i = 0; i < doubles_length; i++)
     {
-      if(amps_Fscanf(file, "%lf ", &recvd_doubles[i]) != 1)
+      if (amps_Fscanf(file, "%lf ", &recvd_doubles[i]) != 1)
       {
-	amps_Printf("ERROR: reading short\n");
-	result |= 1;
-	amps_Exit(1);
+        amps_Printf("ERROR: reading short\n");
+        result |= 1;
+        amps_Exit(1);
       }
     }
 
     for (i = 0; i < floats_length; i++)
     {
-      if(amps_Fscanf(file, "%f ", &recvd_floats[i]) != 1)
+      if (amps_Fscanf(file, "%f ", &recvd_floats[i]) != 1)
       {
-	amps_Printf("ERROR: reading short\n");
-	result |= 1;
+        amps_Printf("ERROR: reading short\n");
+        result |= 1;
       }
     }
 
@@ -196,7 +198,7 @@ int main(int argc, char *argv[])
 {
   char *in_filename = "test9.input";
   char *out_filename = "test10.input";
-  
+
   amps_File file;
   amps_Invoice recv_invoice;
 
@@ -235,7 +237,7 @@ int main(int argc, char *argv[])
 
   me = amps_Rank(amps_CommWorld);
 
-  if(me == 0)
+  if (me == 0)
   {
     FILE* test_file;
 
@@ -246,8 +248,8 @@ int main(int argc, char *argv[])
     fprintf(test_file, "4 10 234 5 6\n");
     fprintf(test_file, "65555 200 234 678 890 6789 2789\n");
     fprintf(test_file, "100000 2789 78 8 1 98 987 98765\n");
-    fprintf(test_file, "12.500000 12.000500 17.400000 679.800000\n"); 
-    fprintf(test_file, "12.500000 0.078000 679.799988 0.500000\n"); 
+    fprintf(test_file, "12.500000 12.000500 17.400000 679.800000\n");
+    fprintf(test_file, "12.500000 0.078000 679.799988 0.500000\n");
 
     fclose(test_file);
   }
diff --git a/pfsimulator/amps/test/src/test12.c b/pfsimulator/amps/test/src/test12.c
index 1ce702053..383fae74e 100644
--- a/pfsimulator/amps/test/src/test12.c
+++ b/pfsimulator/amps/test/src/test12.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*
  * This is a simple "ring" test.  It send a message from the host
  * to all the nodes.
diff --git a/pfsimulator/amps/test/src/test13.c b/pfsimulator/amps/test/src/test13.c
index 8b81b2791..5bea64fda 100644
--- a/pfsimulator/amps/test/src/test13.c
+++ b/pfsimulator/amps/test/src/test13.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*
  * This is a simple "ring" test.  It send a message from the host
  * to all the nodes.
diff --git a/pfsimulator/amps/test/src/test14.c b/pfsimulator/amps/test/src/test14.c
index c5e398a0a..fd89ebfe2 100644
--- a/pfsimulator/amps/test/src/test14.c
+++ b/pfsimulator/amps/test/src/test14.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*
  * This is a simple "ring" test.  It send a message from the host
  * to all the nodes
diff --git a/pfsimulator/amps/test/src/test15.c b/pfsimulator/amps/test/src/test15.c
index 7e5a912d0..1cc215e48 100644
--- a/pfsimulator/amps/test/src/test15.c
+++ b/pfsimulator/amps/test/src/test15.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*
  * Tests for layer ghost exchange.
  */
diff --git a/pfsimulator/amps/test/src/test16.c b/pfsimulator/amps/test/src/test16.c
index 97f2b5699..14d66540d 100644
--- a/pfsimulator/amps/test/src/test16.c
+++ b/pfsimulator/amps/test/src/test16.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*
  * Test for ghost exchange.
  */
diff --git a/pfsimulator/amps/test/src/test17.c b/pfsimulator/amps/test/src/test17.c
index 4e28cd27c..49e55b8cd 100644
--- a/pfsimulator/amps/test/src/test17.c
+++ b/pfsimulator/amps/test/src/test17.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "amps.h"
 #include "amps_test.h"
@@ -38,7 +38,7 @@ int main(int argc, char *argv[])
   amps_File file;
   amps_Invoice recv_invoice;
 
-  /* Number of times to execute SFBcast; default test is 1 */  
+  /* Number of times to execute SFBcast; default test is 1 */
   int loop = 1;
 
   int me;
@@ -68,13 +68,13 @@ int main(int argc, char *argv[])
 
   me = amps_Rank(amps_CommWorld);
 
-  if(me == 0)
+  if (me == 0)
   {
     FILE* test_file;
 
     test_file = fopen(filename, "wb");
 
-    for(unsigned char i = 0; i < length; i++)
+    for (unsigned char i = 0; i < length; i++)
     {
       fwrite(&i, 1, 1, test_file);
     }
@@ -92,16 +92,16 @@ int main(int argc, char *argv[])
 
     amps_SFBCast(amps_CommWorld, file, recv_invoice);
 
-    for(unsigned char i = 0; i < length; i++)
+    for (unsigned char i = 0; i < length; i++)
     {
       if (buffer[i] != i)
       {
-	amps_Printf("ERROR - byte buffers do not match\n");
-	result = 1;
-	break;
+        amps_Printf("ERROR - byte buffers do not match\n");
+        result = 1;
+        break;
       }
     }
-    
+
     amps_SFclose(file);
   }
 
diff --git a/pfsimulator/amps/test/src/test18.c b/pfsimulator/amps/test/src/test18.c
index 455dc81ba..4484d9c6f 100644
--- a/pfsimulator/amps/test/src/test18.c
+++ b/pfsimulator/amps/test/src/test18.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*
  * Tests for layer ghost exchange.
  */
diff --git a/pfsimulator/amps/test/src/test2.c b/pfsimulator/amps/test/src/test2.c
index 7ba11e0d3..2783ab12c 100644
--- a/pfsimulator/amps/test/src/test2.c
+++ b/pfsimulator/amps/test/src/test2.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*
  * This is a simple "ring" test.  It send a message from the host
  * to all the nodes
diff --git a/pfsimulator/amps/test/src/test3.c b/pfsimulator/amps/test/src/test3.c
index cbcc7b495..8f1ea1c72 100644
--- a/pfsimulator/amps/test/src/test3.c
+++ b/pfsimulator/amps/test/src/test3.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*
  * This is a simple "ring" test.  It send a message from the host
  * to all the nodes.  The message contains every supported datatype.
diff --git a/pfsimulator/amps/test/src/test4.c b/pfsimulator/amps/test/src/test4.c
index b01c94ac1..11eb9460e 100644
--- a/pfsimulator/amps/test/src/test4.c
+++ b/pfsimulator/amps/test/src/test4.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 
 #include "amps.h"
@@ -81,10 +81,10 @@ int main(int argc, char *argv[])
       result = strcmp(recvd_string, string);
       if (result)
       {
-	result |= 1;
+        result |= 1;
         amps_Printf("############## ERROR - strings don't match\n");
       }
-	
+
 
       if (loop != temp)
       {
diff --git a/pfsimulator/amps/test/src/test5.c b/pfsimulator/amps/test/src/test5.c
index ca6518f85..d1f1a6358 100644
--- a/pfsimulator/amps/test/src/test5.c
+++ b/pfsimulator/amps/test/src/test5.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 #include "amps_test.h"
 
diff --git a/pfsimulator/amps/test/src/test6.c b/pfsimulator/amps/test/src/test6.c
index c50088d22..5750c26de 100644
--- a/pfsimulator/amps/test/src/test6.c
+++ b/pfsimulator/amps/test/src/test6.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*
  * This is a simple "ring" test.  It send a message from the host
  * to all the nodes.
diff --git a/pfsimulator/amps/test/src/test7.c b/pfsimulator/amps/test/src/test7.c
index 7f6907ace..45ce3a5c0 100644
--- a/pfsimulator/amps/test/src/test7.c
+++ b/pfsimulator/amps/test/src/test7.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*
  * This test prints out information size and rank information on all
  * the nodes and then exits
diff --git a/pfsimulator/amps/test/src/test8.c b/pfsimulator/amps/test/src/test8.c
index 9475947d3..c108dbde8 100644
--- a/pfsimulator/amps/test/src/test8.c
+++ b/pfsimulator/amps/test/src/test8.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /* This is a test string */
 
 #include "amps.h"
@@ -58,7 +58,7 @@ int main(int argc, char *argv[])
 
   int me = amps_Rank(amps_CommWorld);
 
-  if(me == 0)
+  if (me == 0)
   {
     FILE* test_file;
 
diff --git a/pfsimulator/amps/test/src/test9.c b/pfsimulator/amps/test/src/test9.c
index f8f6a59fb..ea2bdc554 100644
--- a/pfsimulator/amps/test/src/test9.c
+++ b/pfsimulator/amps/test/src/test9.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /* The following needs to be in the input file test9.input */
 /*
  * 11
@@ -92,7 +92,7 @@ int main(int argc, char *argv[])
 
   me = amps_Rank(amps_CommWorld);
 
-  if(me == 0)
+  if (me == 0)
   {
     FILE* test_file;
 
@@ -103,8 +103,8 @@ int main(int argc, char *argv[])
     fprintf(test_file, "4 10 234 5 6\n");
     fprintf(test_file, "65555 200 234 678 890 6789 2789\n");
     fprintf(test_file, "100000 2789 78 8 1 98 987 98765\n");
-    fprintf(test_file, "12.500000 12.000500 17.400000 679.800000\n"); 
-    fprintf(test_file, "12.500000 0.078000 679.799988 0.500000\n"); 
+    fprintf(test_file, "12.500000 12.000500 17.400000 679.800000\n");
+    fprintf(test_file, "12.500000 0.078000 679.799988 0.500000\n");
 
     fclose(test_file);
   }
@@ -138,7 +138,7 @@ int main(int argc, char *argv[])
                   string, recvd_string);
       result |= 1;
     }
-    
+
     for (i = 0; i < shorts_length; i++)
       if (shorts[i] != recvd_shorts[i])
       {
@@ -178,7 +178,7 @@ int main(int argc, char *argv[])
                     floats[i], recvd_floats[i]);
         result |= 1;
       }
-    
+
     amps_FreeInvoice(recv_invoice);
   }
 
diff --git a/pfsimulator/amps/test/src/time4.c b/pfsimulator/amps/test/src/time4.c
index 6789d5f4e..05db0d971 100644
--- a/pfsimulator/amps/test/src/time4.c
+++ b/pfsimulator/amps/test/src/time4.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <stdio.h>
 #include "amps.h"
diff --git a/pfsimulator/amps/test/src/timing1.c b/pfsimulator/amps/test/src/timing1.c
index db9ec0893..5bbecc480 100644
--- a/pfsimulator/amps/test/src/timing1.c
+++ b/pfsimulator/amps/test/src/timing1.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <amps.h>
 
 #define method "Shared Memory Put"
diff --git a/pfsimulator/amps/win32/amps.h b/pfsimulator/amps/win32/amps.h
index b7bdd4170..84777f3b3 100644
--- a/pfsimulator/amps/win32/amps.h
+++ b/pfsimulator/amps/win32/amps.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #ifndef _AMPS_HEADER
 #define _AMPS_HEADER
 
@@ -265,140 +265,140 @@ extern amps_Invoice *amps_PtrSyncInvoice;
 /*---------------------------------------------------------------------------*/
 /* Functions to for align                                                    */
 /*---------------------------------------------------------------------------*/
-#define AMPS_ALIGN(type, src, dest, len, stride) \
-  ((sizeof(type) -                               \
-    ((unsigned long)(dest) % sizeof(type)))      \
-   % sizeof(type));
+#define AMPS_ALIGN(type, src, dest, len, stride)       \
+        ((sizeof(type) -                               \
+          ((unsigned long)(dest) % sizeof(type)))      \
+         % sizeof(type));
 
 #define AMPS_CALL_CHAR_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(char, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(char, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_SHORT_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(short, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(short, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_INT_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(int, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(int, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_LONG_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(long, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(long, (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_FLOAT_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(float, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(float, (_src), (_dest), (_len), (_stride))
 
 /* SGS warning!!!!!!:  since malloc returns 4 byte alinged things we should align
  * in a similiar way */
 
 #define AMPS_CALL_DOUBLE_ALIGN(_comm, _src, _dest, _len, _stride) \
-  AMPS_ALIGN(float, (_src), (_dest), (_len), (_stride))
+        AMPS_ALIGN(float, (_src), (_dest), (_len), (_stride))
 
 /*---------------------------------------------------------------------------*/
 /* Functions to for sizeof                                                   */
 /*---------------------------------------------------------------------------*/
 #define AMPS_SIZEOF(len, stride, size) \
-  ((len) * (size))
+        ((len) * (size))
 
 #define AMPS_CALL_CHAR_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(char))
+        AMPS_SIZEOF((_len), (_stride), sizeof(char))
 
 #define AMPS_CALL_SHORT_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(short))
+        AMPS_SIZEOF((_len), (_stride), sizeof(short))
 
 #define AMPS_CALL_INT_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(int))
+        AMPS_SIZEOF((_len), (_stride), sizeof(int))
 
 #define AMPS_CALL_LONG_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(long))
+        AMPS_SIZEOF((_len), (_stride), sizeof(long))
 
 #define AMPS_CALL_FLOAT_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(float))
+        AMPS_SIZEOF((_len), (_stride), sizeof(float))
 
 #define AMPS_CALL_DOUBLE_SIZEOF(_comm, _src, _dest, _len, _stride) \
-  AMPS_SIZEOF((_len), (_stride), sizeof(double))
+        AMPS_SIZEOF((_len), (_stride), sizeof(double))
 
 /*---------------------------------------------------------------------------*/
 /*---------------------------------------------------------------------------*/
-#define AMPS_CONVERT_OUT(type, cvt, comm, src, dest, len, stride)                                         \
-  {                                                                                                       \
-    type *ptr_src, *ptr_dest;                                                                             \
-    if ((char*)(src) != (char*)(dest))                                                                    \
-      if ((stride) == 1)                                                                                  \
-        memcpy((dest), (src), (len) * sizeof(type));                                                      \
-      else                                                                                                \
-        for (ptr_src = (type*)(src), ptr_dest = (type*)(dest); ptr_src < (type*)(src) + (len) * (stride); \
-             ptr_src += (stride), ptr_dest++)                                                             \
-          *ptr_dest = *ptr_src;                                                                           \
-  }
+#define AMPS_CONVERT_OUT(type, cvt, comm, src, dest, len, stride)                                               \
+        {                                                                                                       \
+          type *ptr_src, *ptr_dest;                                                                             \
+          if ((char*)(src) != (char*)(dest))                                                                    \
+          if ((stride) == 1)                                                                                    \
+          memcpy((dest), (src), (len) * sizeof(type));                                                          \
+          else                                                                                                  \
+          for (ptr_src = (type*)(src), ptr_dest = (type*)(dest); ptr_src < (type*)(src) + (len) * (stride);     \
+               ptr_src += (stride), ptr_dest++)                                                                 \
+          *ptr_dest = *ptr_src;                                                                                 \
+        }
 
 #define AMPS_CALL_CHAR_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(char, ctohc, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(char, ctohc, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_SHORT_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(short, ctohs, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(short, ctohs, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_INT_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(int, ctohi, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(int, ctohi, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_LONG_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(long, ctohl, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(long, ctohl, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_FLOAT_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(float, ctohf, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(float, ctohf, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_DOUBLE_OUT(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_OUT(double, ctohd, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_OUT(double, ctohd, (_comm), (_src), (_dest), (_len), (_stride))
 
 
 
-#define AMPS_CONVERT_IN(type, cvt, comm, src, dest, len, stride)                                                \
-  {                                                                                                             \
-    type *ptr_src, *ptr_dest;                                                                                   \
-    if ((char*)(src) != (char*)(dest))                                                                          \
-      if ((stride) == 1)                                                                                        \
-        memcpy((dest), (src), (len) * sizeof(type));                                                            \
-      else                                                                                                      \
-        for (ptr_src = (type*)(src), (ptr_dest) = (type*)(dest); (ptr_dest) < (type*)(dest) + (len) * (stride); \
-             (ptr_src)++, (ptr_dest) += (stride))                                                               \
-          *(ptr_dest) = *(ptr_src);                                                                             \
-  }
+#define AMPS_CONVERT_IN(type, cvt, comm, src, dest, len, stride)                                                      \
+        {                                                                                                             \
+          type *ptr_src, *ptr_dest;                                                                                   \
+          if ((char*)(src) != (char*)(dest))                                                                          \
+          if ((stride) == 1)                                                                                          \
+          memcpy((dest), (src), (len) * sizeof(type));                                                                \
+          else                                                                                                        \
+          for (ptr_src = (type*)(src), (ptr_dest) = (type*)(dest); (ptr_dest) < (type*)(dest) + (len) * (stride);     \
+               (ptr_src)++, (ptr_dest) += (stride))                                                                   \
+          *(ptr_dest) = *(ptr_src);                                                                                   \
+        }
 
 
 #define AMPS_CALL_CHAR_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(char, htocc, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(char, htocc, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_SHORT_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(short, htocs, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(short, htocs, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_INT_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(int, htoci, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(int, htoci, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_LONG_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(long, htocl, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(long, htocl, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_FLOAT_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(float, htocf, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(float, htocf, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CALL_DOUBLE_IN(_comm, _src, _dest, _len, _stride) \
-  AMPS_CONVERT_IN(double, htocd, (_comm), (_src), (_dest), (_len), (_stride))
+        AMPS_CONVERT_IN(double, htocd, (_comm), (_src), (_dest), (_len), (_stride))
 
 #define AMPS_CHECK_OVERLAY(_type, _comm) 1
 
 #define AMPS_CHAR_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(char, _comm)
+        AMPS_CHECK_OVERLAY(char, _comm)
 
 #define AMPS_SHORT_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(short, _comm)
+        AMPS_CHECK_OVERLAY(short, _comm)
 
 #define AMPS_INT_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(int, _comm)
+        AMPS_CHECK_OVERLAY(int, _comm)
 
 #define AMPS_LONG_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(long, _comm)
+        AMPS_CHECK_OVERLAY(long, _comm)
 
 #define AMPS_FLOAT_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(float, _comm)
+        AMPS_CHECK_OVERLAY(float, _comm)
 
 #define AMPS_DOUBLE_OVERLAY(_comm) \
-  AMPS_CHECK_OVERLAY(double, _comm)
+        AMPS_CHECK_OVERLAY(double, _comm)
 
 
 /*---------------------------------------------------------------------------*/
@@ -411,20 +411,20 @@ extern amps_Invoice *amps_PtrSyncInvoice;
 /* Internal macros used to clear buffer and letter spaces.                   */
 /*---------------------------------------------------------------------------*/
 
-#define AMPS_CLEAR_INVOICE(invoice)                     \
-  {                                                     \
-    (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED; \
-    amps_ClearInvoice(invoice);                         \
-  }
+#define AMPS_CLEAR_INVOICE(invoice)                           \
+        {                                                     \
+          (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED; \
+          amps_ClearInvoice(invoice);                         \
+        }
 
-#define AMPS_PACK_FREE_LETTER(comm, invoice, amps_letter) \
-  if ((invoice)->combuf_flags & AMPS_INVOICE_OVERLAYED)   \
-    (invoice)->combuf_flags |= AMPS_INVOICE_ALLOCATED;    \
-  else                                                    \
-  {                                                       \
-    (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED;   \
-    amps_free((comm), (amps_letter));                     \
-  }                                                       \
+#define AMPS_PACK_FREE_LETTER(comm, invoice, amps_letter)       \
+        if ((invoice)->combuf_flags & AMPS_INVOICE_OVERLAYED)   \
+        (invoice)->combuf_flags |= AMPS_INVOICE_ALLOCATED;      \
+        else                                                    \
+        {                                                       \
+          (invoice)->combuf_flags &= ~AMPS_INVOICE_ALLOCATED;   \
+          amps_free((comm), (amps_letter));                     \
+        }                                                       \
 
 
 /*****************************************************************************
@@ -444,29 +444,29 @@ extern amps_Invoice *amps_PtrSyncInvoice;
 /*---------------------------------------------------------------------------*/
 
 #define amps_WriteChar(file, ptr, len) \
-  fwrite((ptr), sizeof(char), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_WriteShort(file, ptr, len) \
-  fwrite((ptr), sizeof(short), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(short), (len), (FILE*)(file))
 
 void amps_WriteInt();
 
 #define amps_WriteLong(file, ptr, len) \
-  fwrite((ptr), sizeof(long), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(long), (len), (FILE*)(file))
 
 void amps_WriteDouble();
 
 
 #define amps_ReadChar(file, ptr, len) \
-  fread((ptr), sizeof(char), (len), (FILE*)(file))
+        fread((ptr), sizeof(char), (len), (FILE*)(file))
 
 #define amps_ReadShort(file, ptr, len) \
-  fread((ptr), sizeof(short), (len), (FILE*)(file))
+        fread((ptr), sizeof(short), (len), (FILE*)(file))
 
 void amps_ReadInt();
 
 #define amps_ReadLong(file, ptr, len) \
-  fread((ptr), sizeof(long), (len), (FILE*)(file))
+        fread((ptr), sizeof(long), (len), (FILE*)(file))
 
 void amps_ReadDouble();
 
@@ -477,10 +477,10 @@ void amps_ReadDouble();
 #if 1
 
 #define amps_TAlloc(type, count) \
-  (type*)_amps_TAlloc(count * sizeof(type), __FILE__, __LINE__)
+        (type*)_amps_TAlloc(count * sizeof(type), __FILE__, __LINE__)
 
 #define amps_CTAlloc(type, count) \
-  (type*)_amps_CTAlloc(count * sizeof(type), __FILE__, __LINE__)
+        (type*)_amps_CTAlloc(count * sizeof(type), __FILE__, __LINE__)
 
 
 /* note: the `else' is required to guarantee termination of the `if' */
@@ -493,10 +493,10 @@ void amps_ReadDouble();
 #else
 
 #define amps_TAlloc(type, count) \
-  ((count) ? (type*)malloc((unsigned int)(sizeof(type) * (count))) : NULL)
+        ((count) ? (type*)malloc((unsigned int)(sizeof(type) * (count))) : NULL)
 
 #define amps_CTAlloc(type, count) \
-  ((count) ? (type*)calloc((unsigned int)(count), (unsigned int)sizeof(type)) : NULL)
+        ((count) ? (type*)calloc((unsigned int)(count), (unsigned int)sizeof(type)) : NULL)
 
 /* note: the `else' is required to guarantee termination of the `if' */
 #define amps_TFree(ptr) if (ptr) free(ptr, amps_arena); else
diff --git a/pfsimulator/amps/win32/amps_allreduce.c b/pfsimulator/amps/win32/amps_allreduce.c
index 4f52d8edc..d2511692c 100644
--- a/pfsimulator/amps/win32/amps_allreduce.c
+++ b/pfsimulator/amps/win32/amps_allreduce.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 #if 0
diff --git a/pfsimulator/amps/win32/amps_bcast.c b/pfsimulator/amps/win32/amps_bcast.c
index a0a3c1298..58f5dc251 100644
--- a/pfsimulator/amps/win32/amps_bcast.c
+++ b/pfsimulator/amps/win32/amps_bcast.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 int
diff --git a/pfsimulator/amps/win32/amps_createinvoice.c b/pfsimulator/amps/win32/amps_createinvoice.c
index 3a0a039b5..f76f852fd 100644
--- a/pfsimulator/amps/win32/amps_createinvoice.c
+++ b/pfsimulator/amps/win32/amps_createinvoice.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <string.h>
 #include <stdarg.h>
 
diff --git a/pfsimulator/amps/win32/amps_exchange.c b/pfsimulator/amps/win32/amps_exchange.c
index bc01eed8e..790a02f81 100644
--- a/pfsimulator/amps/win32/amps_exchange.c
+++ b/pfsimulator/amps/win32/amps_exchange.c
@@ -1,43 +1,43 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
-#define AMPS_COPY(type, src, len, src_stride, dest, dest_stride)   \
-  {                                                                \
-    type *ptr_src, *ptr_dest;                                      \
-    if (((src_stride) == 1) && ((dest_stride == 1)))               \
-      memcpy((dest), (src), (len) * sizeof(type));                 \
-    else                                                           \
-      for (ptr_src = (type*)(src), (ptr_dest) = (type*)(dest);     \
-           (ptr_dest) < (type*)(dest) + (len) * (dest_stride);     \
-           (ptr_src) += (src_stride), (ptr_dest) += (dest_stride)) \
-        *(ptr_dest) = *(ptr_src);                                  \
-  }
+#define AMPS_COPY(type, src, len, src_stride, dest, dest_stride)         \
+        {                                                                \
+          type *ptr_src, *ptr_dest;                                      \
+          if (((src_stride) == 1) && ((dest_stride == 1)))               \
+          memcpy((dest), (src), (len) * sizeof(type));                   \
+          else                                                           \
+          for (ptr_src = (type*)(src), (ptr_dest) = (type*)(dest);       \
+               (ptr_dest) < (type*)(dest) + (len) * (dest_stride);       \
+               (ptr_src) += (src_stride), (ptr_dest) += (dest_stride))   \
+          *(ptr_dest) = *(ptr_src);                                      \
+        }
 
 
 void amps_vector_copy(type, items, dim, ptr_src, len, ptr_dst, dst_stride)
diff --git a/pfsimulator/amps/win32/amps_ffopen.c b/pfsimulator/amps/win32/amps_ffopen.c
index 2dea02388..38911765e 100644
--- a/pfsimulator/amps/win32/amps_ffopen.c
+++ b/pfsimulator/amps/win32/amps_ffopen.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <string.h>
 #include <stdio.h>
 
diff --git a/pfsimulator/amps/win32/amps_finalize.c b/pfsimulator/amps/win32/amps_finalize.c
index fd98bc561..011393a73 100644
--- a/pfsimulator/amps/win32/amps_finalize.c
+++ b/pfsimulator/amps/win32/amps_finalize.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <malloc.h>
 
 #include "amps.h"
diff --git a/pfsimulator/amps/win32/amps_fopen.c b/pfsimulator/amps/win32/amps_fopen.c
index 8ef7f14c9..1664a45ea 100644
--- a/pfsimulator/amps/win32/amps_fopen.c
+++ b/pfsimulator/amps/win32/amps_fopen.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 amps_File amps_Fopen(filename, type)
diff --git a/pfsimulator/amps/win32/amps_init.c b/pfsimulator/amps/win32/amps_init.c
index cd0c18084..9161de385 100644
--- a/pfsimulator/amps/win32/amps_init.c
+++ b/pfsimulator/amps/win32/amps_init.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 #undef main
diff --git a/pfsimulator/amps/win32/amps_io.c b/pfsimulator/amps/win32/amps_io.c
index 015cbb8fa..d03c1e0d1 100644
--- a/pfsimulator/amps/win32/amps_io.c
+++ b/pfsimulator/amps/win32/amps_io.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 void amps_ScanChar(file, data, len, stride)
diff --git a/pfsimulator/amps/win32/amps_irecv.c b/pfsimulator/amps/win32/amps_irecv.c
index 6953f9a90..70651758a 100644
--- a/pfsimulator/amps/win32/amps_irecv.c
+++ b/pfsimulator/amps/win32/amps_irecv.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 char *amps_recv(src)
diff --git a/pfsimulator/amps/win32/amps_newpackage.c b/pfsimulator/amps/win32/amps_newpackage.c
index 3efc4947e..fa0938c16 100644
--- a/pfsimulator/amps/win32/amps_newpackage.c
+++ b/pfsimulator/amps/win32/amps_newpackage.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 amps_Package amps_NewPackage(amps_Comm     comm,
diff --git a/pfsimulator/amps/win32/amps_pack.c b/pfsimulator/amps/win32/amps_pack.c
index 39157a379..be080ede6 100644
--- a/pfsimulator/amps/win32/amps_pack.c
+++ b/pfsimulator/amps/win32/amps_pack.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <string.h>
 #include <stdarg.h>
 
diff --git a/pfsimulator/amps/win32/amps_proto.h b/pfsimulator/amps/win32/amps_proto.h
index b211728e0..1e1b11619 100644
--- a/pfsimulator/amps/win32/amps_proto.h
+++ b/pfsimulator/amps/win32/amps_proto.h
@@ -1,138 +1,138 @@
 /* _rand48.c */
-void _dorand48 (unsigned short xseed [3 ]);
+void _dorand48(unsigned short xseed[3 ]);
 
 /* amps_allreduce.c */
-int amps_ReduceOperation (amps_Comm comm, amps_Invoice invoice, char *buf_dest, char *buf_src, int operation);
-int amps_AllReduce (amps_Comm comm, amps_Invoice invoice, int operation);
+int amps_ReduceOperation(amps_Comm comm, amps_Invoice invoice, char *buf_dest, char *buf_src, int operation);
+int amps_AllReduce(amps_Comm comm, amps_Invoice invoice, int operation);
 
 /* amps_bcast.c */
-int amps_BCast (amps_Comm comm, int source, amps_Invoice invoice);
+int amps_BCast(amps_Comm comm, int source, amps_Invoice invoice);
 
 /* amps_clear.c */
-void amps_ClearInvoice (amps_Invoice inv);
+void amps_ClearInvoice(amps_Invoice inv);
 
 /* amps_createinvoice.c */
-int amps_CreateInvoice (amps_Comm comm, amps_Invoice inv);
+int amps_CreateInvoice(amps_Comm comm, amps_Invoice inv);
 
 /* amps_exchange.c */
-void _amps_wait_exchange (amps_Handle handle);
-amps_Handle amps_IExchangePackage (amps_Package package);
+void _amps_wait_exchange(amps_Handle handle);
+amps_Handle amps_IExchangePackage(amps_Package package);
 
 /* amps_ffopen.c */
-amps_File amps_FFopen (amps_Comm comm, char *filename, char *type, long size);
+amps_File amps_FFopen(amps_Comm comm, char *filename, char *type, long size);
 
 /* amps_finalize.c */
-int amps_Finalize (void);
+int amps_Finalize(void);
 
 /* amps_find_powers.c */
-void amps_FindPowers (int N, int *log, int *Nnext, int *Nprev);
+void amps_FindPowers(int N, int *log, int *Nnext, int *Nprev);
 
 /* amps_fopen.c */
-amps_File amps_Fopen (char *filename, char *type);
+amps_File amps_Fopen(char *filename, char *type);
 
 /* amps_init.c */
-int amps_Init (int *argc, char **argv[]);
-unsigned amps_main (void *arg);
-int main (int argc, char *argv []);
-void *_amps_CTAlloc (int count, char *filename, int line);
-void *_amps_TAlloc (int count, char *filename, int line);
+int amps_Init(int *argc, char **argv[]);
+unsigned amps_main(void *arg);
+int main(int argc, char *argv []);
+void *_amps_CTAlloc(int count, char *filename, int line);
+void *_amps_TAlloc(int count, char *filename, int line);
 
 /* amps_invoice.c */
-void amps_AppendInvoice (amps_Invoice *invoice, amps_Invoice append_invoice);
-amps_Invoice amps_new_empty_invoice (void);
-int amps_FreeInvoice (amps_Invoice inv);
-int amps_add_invoice (amps_Invoice *inv, int ignore, int type, int len_type, int len, int *ptr_len, int stride_type, int stride, int *ptr_stride, int dim_type, int dim, int *ptr_dim, int data_type, void *data);
-amps_Invoice amps_NewInvoice (const char *fmt0, ...);
-int amps_num_package_items (amps_Invoice inv);
+void amps_AppendInvoice(amps_Invoice *invoice, amps_Invoice append_invoice);
+amps_Invoice amps_new_empty_invoice(void);
+int amps_FreeInvoice(amps_Invoice inv);
+int amps_add_invoice(amps_Invoice *inv, int ignore, int type, int len_type, int len, int *ptr_len, int stride_type, int stride, int *ptr_stride, int dim_type, int dim, int *ptr_dim, int data_type, void *data);
+amps_Invoice amps_NewInvoice(const char *fmt0, ...);
+int amps_num_package_items(amps_Invoice inv);
 
 /* amps_io.c */
-void amps_ScanChar (amps_File file, char *data, int len, int stride);
-void amps_ScanShort (amps_File file, short *data, int len, int stride);
-void amps_ScanInt (amps_File file, int *data, int len, int stride);
-void amps_ScanLong (amps_File file, long *data, int len, int stride);
-void amps_ScanFloat (amps_File file, float *data, int len, int stride);
-void amps_ScanDouble (amps_File file, double *data, int len, int stride);
-void amps_WriteDouble (amps_File file, double *ptr, int len);
-void amps_WriteInt (amps_File file, int *ptr, int len);
-void amps_ReadDouble (amps_File file, double *ptr, int len);
-void amps_ReadInt (amps_File file, int *ptr, int len);
+void amps_ScanChar(amps_File file, char *data, int len, int stride);
+void amps_ScanShort(amps_File file, short *data, int len, int stride);
+void amps_ScanInt(amps_File file, int *data, int len, int stride);
+void amps_ScanLong(amps_File file, long *data, int len, int stride);
+void amps_ScanFloat(amps_File file, float *data, int len, int stride);
+void amps_ScanDouble(amps_File file, double *data, int len, int stride);
+void amps_WriteDouble(amps_File file, double *ptr, int len);
+void amps_WriteInt(amps_File file, int *ptr, int len);
+void amps_ReadDouble(amps_File file, double *ptr, int len);
+void amps_ReadInt(amps_File file, int *ptr, int len);
 
 /* amps_irecv.c */
-char *amps_recv (int src);
-amps_Handle amps_IRecv (amps_Comm comm, int source, amps_Invoice invoice);
+char *amps_recv(int src);
+amps_Handle amps_IRecv(amps_Comm comm, int source, amps_Invoice invoice);
 
 /* amps_newhandle.c */
-amps_Handle amps_NewHandle (amps_Comm comm, int id, amps_Invoice invoice, amps_Package package);
+amps_Handle amps_NewHandle(amps_Comm comm, int id, amps_Invoice invoice, amps_Package package);
 
 /* amps_newpackage.c */
-amps_Package amps_NewPackage (amps_Comm comm, int num_send, int *dest, amps_Invoice *send_invoices, int num_recv, int *src, amps_Invoice *recv_invoices);
-void amps_FreePackage (amps_Package package);
+amps_Package amps_NewPackage(amps_Comm comm, int num_send, int *dest, amps_Invoice *send_invoices, int num_recv, int *src, amps_Invoice *recv_invoices);
+void amps_FreePackage(amps_Package package);
 
 /* amps_pack.c */
-int amps_pack (amps_Comm comm, amps_Invoice inv, char **buffer);
+int amps_pack(amps_Comm comm, amps_Invoice inv, char **buffer);
 
 /* amps_recv.c */
-char *amps_recvb (int src);
-int amps_Recv (amps_Comm comm, int source, amps_Invoice invoice);
+char *amps_recvb(int src);
+int amps_Recv(amps_Comm comm, int source, amps_Invoice invoice);
 
 /* amps_send.c */
-int amps_xsend (char *buffer, int dest);
-int amps_Send (amps_Comm comm, int dest, amps_Invoice invoice);
+int amps_xsend(char *buffer, int dest);
+int amps_Send(amps_Comm comm, int dest, amps_Invoice invoice);
 
 /* amps_sfbcast.c */
-int amps_SFBCast (amps_Comm comm, amps_File file, amps_Invoice invoice);
+int amps_SFBCast(amps_Comm comm, amps_File file, amps_Invoice invoice);
 
 /* amps_sfclose.c */
-int amps_SFclose (amps_File file);
+int amps_SFclose(amps_File file);
 
 /* amps_sfopen.c */
-amps_File amps_SFopen (char *filename, char *type);
+amps_File amps_SFopen(char *filename, char *type);
 
 /* amps_sizeofinvoice.c */
-int amps_sizeof_invoice (amps_Comm comm, amps_Invoice inv);
+int amps_sizeof_invoice(amps_Comm comm, amps_Invoice inv);
 
 /* amps_sync.c */
-int amps_Sync (amps_Comm comm);
+int amps_Sync(amps_Comm comm);
 
 /* amps_test.c */
-int amps_Test (amps_Handle handle);
+int amps_Test(amps_Handle handle);
 
 /* amps_unpack.c */
-int amps_unpack (amps_Comm comm, amps_Invoice inv, char *buffer);
+int amps_unpack(amps_Comm comm, amps_Invoice inv, char *buffer);
 
 /* amps_vector.c */
-void amps_vector_out (amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
-void amps_vector_in (amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
-int amps_vector_align (amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
-int amps_vector_sizeof_buffer (amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
-int amps_vector_sizeof_local (amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
+void amps_vector_out(amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
+void amps_vector_in(amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
+int amps_vector_align(amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
+int amps_vector_sizeof_buffer(amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
+int amps_vector_sizeof_local(amps_Comm comm, int type, char **data, char **buf_ptr, int dim, int *len, int *stride);
 
 /* amps_wait.c */
-int amps_Wait (amps_Handle handle);
+int amps_Wait(amps_Handle handle);
 
 /* drand48.c */
-double drand48 (void);
+double drand48(void);
 
 /* erand48.c */
-double erand48 (unsigned short xseed [3 ]);
+double erand48(unsigned short xseed[3 ]);
 
 /* jrand48.c */
-long jrand48 (unsigned short xseed [3 ]);
+long jrand48(unsigned short xseed[3 ]);
 
 /* lrand48.c */
-long lrand48 (void);
+long lrand48(void);
 
 /* mrand48.c */
-long mrand48 (void);
+long mrand48(void);
 
 /* nrand48.c */
-long nrand48 (unsigned short xseed [3 ]);
+long nrand48(unsigned short xseed[3 ]);
 
 /* srand48.c */
-void srand48 (long seed);
+void srand48(long seed);
 
 /* unix_port.c */
-double d_sign (doublereal *a, doublereal *b);
+double d_sign(doublereal *a, doublereal *b);
 
 /* amps_print.c */
 FILE* amps_SetConsole(FILE* stream);
diff --git a/pfsimulator/amps/win32/amps_recv.c b/pfsimulator/amps/win32/amps_recv.c
index 9b37e7773..1c5d81066 100644
--- a/pfsimulator/amps/win32/amps_recv.c
+++ b/pfsimulator/amps/win32/amps_recv.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 char *amps_recvb(src)
diff --git a/pfsimulator/amps/win32/amps_send.c b/pfsimulator/amps/win32/amps_send.c
index d80c3fcfc..e3ff59551 100644
--- a/pfsimulator/amps/win32/amps_send.c
+++ b/pfsimulator/amps/win32/amps_send.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 int amps_xsend(buffer, dest)
diff --git a/pfsimulator/amps/win32/amps_sizeofinvoice.c b/pfsimulator/amps/win32/amps_sizeofinvoice.c
index 3fddabb16..6f8993956 100644
--- a/pfsimulator/amps/win32/amps_sizeofinvoice.c
+++ b/pfsimulator/amps/win32/amps_sizeofinvoice.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 int amps_sizeof_invoice(comm, inv)
diff --git a/pfsimulator/amps/win32/amps_sync.c b/pfsimulator/amps/win32/amps_sync.c
index bba8d2f2c..1b238a5ec 100644
--- a/pfsimulator/amps/win32/amps_sync.c
+++ b/pfsimulator/amps/win32/amps_sync.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <amps.h>
 
 int amps_Sync(amps_Comm comm)
diff --git a/pfsimulator/amps/win32/amps_test.c b/pfsimulator/amps/win32/amps_test.c
index 14414f12b..a85c7987a 100644
--- a/pfsimulator/amps/win32/amps_test.c
+++ b/pfsimulator/amps/win32/amps_test.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 int amps_Test(handle)
diff --git a/pfsimulator/amps/win32/amps_unpack.c b/pfsimulator/amps/win32/amps_unpack.c
index 9d9cd2d23..2eba81a15 100644
--- a/pfsimulator/amps/win32/amps_unpack.c
+++ b/pfsimulator/amps/win32/amps_unpack.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <string.h>
 #include <stdarg.h>
 
diff --git a/pfsimulator/amps/win32/amps_vector.c b/pfsimulator/amps/win32/amps_vector.c
index 65d1e15ca..57f7e6eee 100644
--- a/pfsimulator/amps/win32/amps_vector.c
+++ b/pfsimulator/amps/win32/amps_vector.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 void amps_vector_out(comm, type, data, buf_ptr, dim, len, stride)
diff --git a/pfsimulator/amps/win32/amps_wait.c b/pfsimulator/amps/win32/amps_wait.c
index ad9fb4048..428d9ca2b 100644
--- a/pfsimulator/amps/win32/amps_wait.c
+++ b/pfsimulator/amps/win32/amps_wait.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "amps.h"
 
 int amps_Wait(handle)
diff --git a/pfsimulator/amps/win32/f2c.h b/pfsimulator/amps/win32/f2c.h
index cfc3ed23e..fabe3c384 100644
--- a/pfsimulator/amps/win32/f2c.h
+++ b/pfsimulator/amps/win32/f2c.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 typedef long int integer;
 typedef char *address;
diff --git a/pfsimulator/amps/win32/freemc.tcl b/pfsimulator/amps/win32/freemc.tcl
index 96040375a..23cfd7bba 100644
--- a/pfsimulator/amps/win32/freemc.tcl
+++ b/pfsimulator/amps/win32/freemc.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pfsimulator/amps/win32/getmc.tcl b/pfsimulator/amps/win32/getmc.tcl
index 9c7de8d5f..a18efdf7d 100644
--- a/pfsimulator/amps/win32/getmc.tcl
+++ b/pfsimulator/amps/win32/getmc.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pfsimulator/amps/win32/run.tcl b/pfsimulator/amps/win32/run.tcl
index 7d173cdb7..a826ee492 100644
--- a/pfsimulator/amps/win32/run.tcl
+++ b/pfsimulator/amps/win32/run.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pfsimulator/amps/win32/strip.tcl b/pfsimulator/amps/win32/strip.tcl
index 5f457af15..e0ec9ab43 100644
--- a/pfsimulator/amps/win32/strip.tcl
+++ b/pfsimulator/amps/win32/strip.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pfsimulator/amps/win32/unix_port.c b/pfsimulator/amps/win32/unix_port.c
index a7522d319..35a937dfa 100644
--- a/pfsimulator/amps/win32/unix_port.c
+++ b/pfsimulator/amps/win32/unix_port.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "f2c.h"
 double d_sign(doublereal *a, doublereal *b)
 {
diff --git a/pfsimulator/amps/win32/values.h b/pfsimulator/amps/win32/values.h
index 4f9d32f73..296865a28 100644
--- a/pfsimulator/amps/win32/values.h
+++ b/pfsimulator/amps/win32/values.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /* This is a substitute for a values.h file */
 
 #define  M_PI            3.14159265358979323846
diff --git a/pfsimulator/kinsol/kinsol.c b/pfsimulator/kinsol/kinsol.c
index 21eb62458..41f5aee15 100644
--- a/pfsimulator/kinsol/kinsol.c
+++ b/pfsimulator/kinsol/kinsol.c
@@ -1054,7 +1054,7 @@ static int KINConstraint(KINMem kin_mem)
 static void KINForcingTerm(KINMem kin_mem, real fnormp)
 {
   real eta_max = POINT9, eta_min = POINTOHOHOHONE,
-    eta_safe = 0.5, linmodel_norm;
+       eta_safe = 0.5, linmodel_norm;
 
   if (etaflag == ETACHOICE1)        /* Choice 1 forcing terms. */
 
@@ -1230,7 +1230,7 @@ static int  KINLineSearch(KINMem kin_mem, real *fnormp, real *f1normp,
 {
   int ret, ivio, nfesav, rladjust = 0;
   real pnorm, ratio, ratio1, slpi, rlmin, rlength, rl, rlmax, rldiff;
-  real rltmp, rlprev=0.0, pt1trl, rllo, rlincr, alpha, beta;
+  real rltmp, rlprev = 0.0, pt1trl, rllo, rlincr, alpha, beta;
   real alpha_cond, beta_cond;
 
   *maxStepTaken = FALSE;
diff --git a/pfsimulator/parflow_exe/main.c b/pfsimulator/parflow_exe/main.c
index 6f1ab4562..81bb4bb27 100644
--- a/pfsimulator/parflow_exe/main.c
+++ b/pfsimulator/parflow_exe/main.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * The main routine
@@ -114,15 +114,19 @@ int main(int argc, char *argv [])
 
 #ifndef NDEBUG
     /*-----------------------------------------------------------------------
-    * Wait for debugger if PARFLOW_DEBUG_RANK environment variable is set
-    *-----------------------------------------------------------------------*/
-    if(getenv("PARFLOW_DEBUG_RANK") != NULL) {
+     * Wait for debugger if PARFLOW_DEBUG_RANK environment variable is set
+     *-----------------------------------------------------------------------*/
+    if (getenv("PARFLOW_DEBUG_RANK") != NULL)
+    {
       const int mpi_debug = atoi(getenv("PARFLOW_DEBUG_RANK"));
-      if(mpi_debug == amps_Rank(amps_CommWorld)){
+      if (mpi_debug == amps_Rank(amps_CommWorld))
+      {
         volatile int i = 0;
         amps_Printf("PARFLOW_DEBUG_RANK environment variable found.\n");
         amps_Printf("Attach debugger to PID %ld (MPI rank %d) and set var i = 1 to continue\n", (long)getpid(), mpi_debug);
-        while(i == 0) {/*  change 'i' in the  debugger  */}
+        while (i == 0)
+        {              /*  change 'i' in the  debugger  */
+        }
       }
       amps_Sync(amps_CommWorld);
     }
@@ -134,21 +138,22 @@ int main(int argc, char *argv [])
 #if defined(PARFLOW_HAVE_KOKKOS)
     kokkosInit();
 #elif defined(PARFLOW_HAVE_CUDA)
-
     /*-----------------------------------------------------------------------
-    * Check CUDA compute capability, set device, and initialize RMM allocator
-    *-----------------------------------------------------------------------*/
+     * Check CUDA compute capability, set device, and initialize RMM allocator
+     *-----------------------------------------------------------------------*/
     {
       // CUDA
       if (!amps_Rank(amps_CommWorld))
       {
-        CUDA_ERR(cudaSetDevice(0));  
-      }else{
+        CUDA_ERR(cudaSetDevice(0));
+      }
+      else
+      {
         int num_devices = 0;
         CUDA_ERR(cudaGetDeviceCount(&num_devices));
         CUDA_ERR(cudaSetDevice(amps_node_rank % num_devices));
       }
-    
+
       int device;
       CUDA_ERR(cudaGetDevice(&device));
 
@@ -161,23 +166,23 @@ int main(int argc, char *argv [])
 
       if (props.major < 6)
       {
-        amps_Printf("\nError: The GPU compute capability %d.%d of %s is not sufficient.\n",props.major,props.minor,props.name);
+        amps_Printf("\nError: The GPU compute capability %d.%d of %s is not sufficient.\n", props.major, props.minor, props.name);
         amps_Printf("\nThe minimum required GPU compute capability is 6.0.\n");
         exit(1);
       }
     }
 #endif // PARFLOW_HAVE_KOKKOS
 
-  /*-----------------------------------------------------------------------
-  * Initialize RMM pool allocator
-  *-----------------------------------------------------------------------*/
+    /*-----------------------------------------------------------------------
+     * Initialize RMM pool allocator
+     *-----------------------------------------------------------------------*/
 #ifdef PARFLOW_HAVE_RMM
-      // RMM
-      rmmOptions_t rmmOptions;
-      rmmOptions.allocation_mode = (rmmAllocationMode_t) (PoolAllocation | CudaManagedMemory);
-      rmmOptions.initial_pool_size = 1; // size = 0 initializes half the device memory
-      rmmOptions.enable_logging = false;
-      RMM_ERR(rmmInitialize(&rmmOptions));
+    // RMM
+    rmmOptions_t rmmOptions;
+    rmmOptions.allocation_mode = (rmmAllocationMode_t)(PoolAllocation | CudaManagedMemory);
+    rmmOptions.initial_pool_size = 1;   // size = 0 initializes half the device memory
+    rmmOptions.enable_logging = false;
+    RMM_ERR(rmmInitialize(&rmmOptions));
 #endif // PARFLOW_HAVE_RMM
 
     wall_clock_time = amps_Clock();
@@ -239,7 +244,7 @@ int main(int argc, char *argv [])
     {
       char filename[2048];
       sprintf(filename, "%s.%06d.etrace", input_name, amps_Rank(MPI_CommWorld));
-      init_tracefile (filename);
+      init_tracefile(filename);
     }
 #endif
 
@@ -486,15 +491,15 @@ int main(int argc, char *argv [])
 #endif
 
   /*-----------------------------------------------------------------------
-  * Shutdown Kokkos
-  *-----------------------------------------------------------------------*/
+   * Shutdown Kokkos
+   *-----------------------------------------------------------------------*/
 #ifdef PARFLOW_HAVE_KOKKOS
   kokkosFinalize();
 #endif
 
   /*-----------------------------------------------------------------------
-  * Shutdown RMM pool allocator
-  *-----------------------------------------------------------------------*/
+   * Shutdown RMM pool allocator
+   *-----------------------------------------------------------------------*/
 #ifdef PARFLOW_HAVE_RMM
   RMM_ERR(rmmFinalize());
 #endif
diff --git a/pfsimulator/parflow_lib/CMakeLists.txt b/pfsimulator/parflow_lib/CMakeLists.txt
index 206a157ca..4cf01e499 100644
--- a/pfsimulator/parflow_lib/CMakeLists.txt
+++ b/pfsimulator/parflow_lib/CMakeLists.txt
@@ -89,6 +89,8 @@ set (SRC_FILES_CONST advect.F
   problem_thermal_conductivity.c
   problem_toposlope_x.c
   problem_toposlope_y.c
+  problem_wc_x.c
+  problem_wc_y.c
   process_grid.c
   random.c
   ratqr.c
@@ -115,6 +117,8 @@ set (SRC_FILES_CONST advect.F
   well.c
   well_package.c
   wells_lb.c
+  reservoir.c
+  reservoir_package.c
   wrf_parflow.c
   write_clm_netcdf.c
   write_parflow_binary.c
@@ -199,7 +203,6 @@ endif(${PARFLOW_HAVE_KOKKOS})
 
   if ( ${PARFLOW_HAVE_CUDA} )
     target_link_libraries(pfsimulator nvToolsExt)
-    target_include_directories(pfsimulator PUBLIC "../third_party/cub")
     target_include_directories(pfsimulator PUBLIC ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
   endif ( ${PARFLOW_HAVE_CUDA} )
 
diff --git a/pfsimulator/parflow_lib/Header.c b/pfsimulator/parflow_lib/Header.c
index 732f19c0f..cdcca4213 100644
--- a/pfsimulator/parflow_lib/Header.c
+++ b/pfsimulator/parflow_lib/Header.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Description
 *
diff --git a/pfsimulator/parflow_lib/Makefile.depend b/pfsimulator/parflow_lib/Makefile.depend
index 513806085..f59cae49e 100644
--- a/pfsimulator/parflow_lib/Makefile.depend
+++ b/pfsimulator/parflow_lib/Makefile.depend
@@ -424,7 +424,7 @@ DEPENDS_22:=\
 	logging.h loops.h matrix.h n_vector.h nl_function_eval.h	\
 	parflow.h parflow_proto.h parflow_proto_f.h pf_module.h		\
 	problem.h problem_bc.h problem_eval.h region.h solver.h		\
-	time_cycle_data.h timing.h vector.h well.h
+	time_cycle_data.h timing.h vector.h well.h reservoir.h
 
 ifeq (${DEPENDS_ON_TEMPLATE_IMPLEMENTATION},yes)
 DEPENDS_22 +=\
diff --git a/pfsimulator/parflow_lib/advect.win32.c b/pfsimulator/parflow_lib/advect.win32.c
index 5fd714c2f..65879354d 100644
--- a/pfsimulator/parflow_lib/advect.win32.c
+++ b/pfsimulator/parflow_lib/advect.win32.c
@@ -5,32 +5,32 @@
 
 #include "f2c.h"
 
-/*BHEADER*********************************************************************
+/*BHEADER**********************************************************************
  *
- * Copyright (c) 1995-2009, Lawrence Livermore National Security,
- * LLC. Produced at the Lawrence Livermore National Laboratory. Written
- * by the Parflow Team (see the CONTRIBUTORS file)
- * <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+ *  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+ *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+ *  by the Parflow Team (see the CONTRIBUTORS file)
+ *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
  *
- * This file is part of Parflow. For details, see
- * http://www.llnl.gov/casc/parflow
+ *  This file is part of Parflow. For details, see
+ *  http://www.llnl.gov/casc/parflow
  *
- * Please read the COPYRIGHT file or Our Notice and the LICENSE file
- * for the GNU Lesser General Public License.
+ *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+ *  for the GNU Lesser General Public License.
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License (as published
- * by the Free Software Foundation) version 2.1 dated February 1999.
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License (as published
+ *  by the Free Software Foundation) version 2.1 dated February 1999.
  *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- * and conditions of the GNU General Public License for more details.
+ *  This program is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+ *  and conditions of the GNU General Public License for more details.
  *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- * USA
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ *  USA
  **********************************************************************EHEADER********************************************************************
  ****/
 /****************************************************************************
@@ -52,7 +52,7 @@ integer *lo, *hi, *dlo, *dhi;
 doublereal *hx, *dt;
 integer *fstord;
 doublereal *sbot, *stop, *sbotp, *sfrt, *sbck, *sleft, *sright, *sfluxz, *
-  dxscr, *dyscr, *dzscr, *dzfrm;
+           dxscr, *dyscr, *dzscr, *dzfrm;
 {
   /* Initialized data */
 
@@ -60,15 +60,15 @@ doublereal *sbot, *stop, *sbotp, *sfrt, *sbck, *sleft, *sright, *sfluxz, *
 
   /* System generated locals */
   integer s_dim1, s_dim2, s_offset, sn_dim1, sn_dim2, sn_offset, uedge_dim1,
-    uedge_dim2, uedge_offset, vedge_dim1, vedge_dim2, vedge_offset,
-    wedge_dim1, wedge_dim2, wedge_offset, phi_dim1, phi_dim2,
-    phi_offset, slx_dim1, slx_offset, sly_dim1, sly_offset, slz_dim1,
-    slz_dim2, slz_offset, sbot_dim1, sbot_offset, stop_dim1,
-    stop_offset, sbotp_dim1, sbotp_offset, sbck_dim1, sbck_offset,
-    sfrt_dim1, sfrt_offset, sleft_offset, sright_offset,
-    sfluxz_offset, dxscr_dim1, dxscr_offset, dyscr_dim1, dyscr_offset,
-    dzscr_dim1, dzscr_offset, dzfrm_dim1, dzfrm_offset, i__1, i__2,
-    i__3;
+          uedge_dim2, uedge_offset, vedge_dim1, vedge_dim2, vedge_offset,
+          wedge_dim1, wedge_dim2, wedge_offset, phi_dim1, phi_dim2,
+          phi_offset, slx_dim1, slx_offset, sly_dim1, sly_offset, slz_dim1,
+          slz_dim2, slz_offset, sbot_dim1, sbot_offset, stop_dim1,
+          stop_offset, sbotp_dim1, sbotp_offset, sbck_dim1, sbck_offset,
+          sfrt_dim1, sfrt_offset, sleft_offset, sright_offset,
+          sfluxz_offset, dxscr_dim1, dxscr_offset, dyscr_dim1, dyscr_offset,
+          dzscr_dim1, dzscr_offset, dzfrm_dim1, dzfrm_offset, i__1, i__2,
+          i__3;
 
   /* Local variables */
   doublereal supw;
@@ -79,9 +79,9 @@ doublereal *sbot, *stop, *sbotp, *sfrt, *sbck, *sleft, *sright, *sfluxz, *
   doublereal dx, dy, dz, phiinv, supw_m__;
   extern /* Subroutine */ int slopez_();
   doublereal supw_p__, dth, dxh, dyh, dzh, dxi, dyi, dzi, cux, cuy,
-    cuz, thi_xhi__, thi_yhi__, thi_zhi__, sux, suy, suz, tlo_xhi__,
-    tlo_yhi__, tlo_zhi__, thi_xlo__, thi_ylo__, thi_zlo__, tlo_xlo__,
-    tlo_ylo__, tlo_zlo__;
+             cuz, thi_xhi__, thi_yhi__, thi_zhi__, sux, suy, suz, tlo_xhi__,
+             tlo_yhi__, tlo_zhi__, thi_xlo__, thi_ylo__, thi_zlo__, tlo_xlo__,
+             tlo_ylo__, tlo_zlo__;
   extern /* Subroutine */ int slopexy_();
 
 /*     ::: argument declarations */
@@ -530,8 +530,8 @@ doublereal *dxscr, *dyscr;
 
   /* System generated locals */
   integer s_dim1, s_dim2, s_offset, slx_dim1, slx_offset, sly_dim1,
-    sly_offset, dxscr_dim1, dxscr_offset, dyscr_dim1, dyscr_offset,
-    i__1, i__2;
+          sly_offset, dxscr_dim1, dxscr_offset, dyscr_dim1, dyscr_offset,
+          i__1, i__2;
   doublereal d__1, d__2, d__3;
 
   /* Builtin functions */
@@ -687,7 +687,7 @@ doublereal *dzscr, *dzfrm;
 
   /* System generated locals */
   integer s_dim1, s_dim2, s_offset, slz_dim1, slz_dim2, slz_offset,
-    dzscr_dim1, dzscr_offset, dzfrm_dim1, dzfrm_offset, i__1, i__2;
+          dzscr_dim1, dzscr_offset, dzfrm_dim1, dzfrm_offset, i__1, i__2;
   doublereal d__1, d__2, d__3;
 
   /* Builtin functions */
diff --git a/pfsimulator/parflow_lib/advection_godunov.c b/pfsimulator/parflow_lib/advection_godunov.c
index 17aed0543..d909483ca 100644
--- a/pfsimulator/parflow_lib/advection_godunov.c
+++ b/pfsimulator/parflow_lib/advection_godunov.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
@@ -134,19 +134,19 @@ void     Godunov(
   SubregionArray   *subregion_array;
 
   Subgrid          *subgrid,
-    *well_subgrid,
-    *tmp_subgrid;
+                   *well_subgrid,
+                   *tmp_subgrid;
   Subregion        *subregion;
   Subvector        *subvector,
-    *subvector_smf,
-    *subvector_scal,
-    *subvector_rhs,
-    *subvector_xvel,
-    *subvector_yvel,
-    *subvector_zvel,
-    *px_sub,
-    *py_sub,
-    *pz_sub;
+                   *subvector_smf,
+                   *subvector_scal,
+                   *subvector_rhs,
+                   *subvector_xvel,
+                   *subvector_yvel,
+                   *subvector_zvel,
+                   *px_sub,
+                   *py_sub,
+                   *pz_sub;
 
   ComputePkg       *compute_pkg;
   Region           *compute_reg = NULL;
@@ -156,11 +156,11 @@ void     Godunov(
   int nx, ny, nz;
   double dx, dy, dz;
   int nx_c, ny_c, nz_c,
-    nx_p, ny_p, nz_p,
-    nx_w, ny_w, nz_w,
-    nx_xv, ny_xv, nz_xv,
-    nx_yv, ny_yv, nz_yv,
-    nx_zv, ny_zv, nz_zv;
+      nx_p, ny_p, nz_p,
+      nx_w, ny_w, nz_w,
+      nx_xv, ny_xv, nz_xv,
+      nx_yv, ny_yv, nz_yv,
+      nx_zv, ny_zv, nz_zv;
   int i, j, k, ci, pi, wi, xi, yi, zi;
   int nx_cells, ny_cells, nz_cells, index, flopest;
   double lambda, decay_factor;
@@ -178,7 +178,7 @@ void     Godunov(
   double dt;
   int fstord;
   double cell_volume, field_sum, total_volume, cell_change,
-    well_stat;
+         well_stat;
   double well_value, input_c, volume, flux, scaled_flux, weight = 0;
   double avg_x, avg_y, avg_z, area_x, area_y, area_z, area_sum;
 
diff --git a/pfsimulator/parflow_lib/axpy.c b/pfsimulator/parflow_lib/axpy.c
index d48cf1f23..0912fd133 100644
--- a/pfsimulator/parflow_lib/axpy.c
+++ b/pfsimulator/parflow_lib/axpy.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *****************************************************************************/
diff --git a/pfsimulator/parflow_lib/backend_mapping.h b/pfsimulator/parflow_lib/backend_mapping.h
index 85acb61b7..91f6e29e8 100644
--- a/pfsimulator/parflow_lib/backend_mapping.h
+++ b/pfsimulator/parflow_lib/backend_mapping.h
@@ -22,21 +22,21 @@
  */
 
 /* PF_COMP_UNIT_TYPE determines the behavior of the NVCC compilation unit and/or OpenMP loops:
-------------------------------------------------------------
-   CUDA
-------------------------------------------------------------
-   1:     NVCC compiler, Unified Memory allocation, Parallel loops on GPUs
-   2:     NVCC compiler, Unified Memory allocation, Sequential loops on host
-   Other: NVCC compiler, Standard heap allocation, Sequential loops on host
-
-
-------------------------------------------------------------
-   OpenMP
-------------------------------------------------------------
-   1:     CXX compiler, Standard heap allocation, Parallel loops on CPU
-   2:     CXX compiler, Standard heap allocation, Sequential loops on CPU
-   Other: CXX compiler, Standard heap allocation, Sequential loops on CPU
-*/
+ * ------------------------------------------------------------
+ * CUDA
+ * ------------------------------------------------------------
+ * 1:     NVCC compiler, Unified Memory allocation, Parallel loops on GPUs
+ * 2:     NVCC compiler, Unified Memory allocation, Sequential loops on host
+ * Other: NVCC compiler, Standard heap allocation, Sequential loops on host
+ *
+ *
+ * ------------------------------------------------------------
+ * OpenMP
+ * ------------------------------------------------------------
+ * 1:     CXX compiler, Standard heap allocation, Parallel loops on CPU
+ * 2:     CXX compiler, Standard heap allocation, Sequential loops on CPU
+ * Other: CXX compiler, Standard heap allocation, Sequential loops on CPU
+ */
 
 /* Include headers depending on the accelerator backend */
 #ifdef PARFLOW_HAVE_KOKKOS
@@ -80,8 +80,8 @@
 
 #define EMPTY()
 #define DEFER(x) x EMPTY()
-#define PASTER(x,y) x ## y
-#define EVALUATOR(x,y) PASTER(x,y)
+#define PASTER(x, y) x ## y
+#define EVALUATOR(x, y) PASTER(x, y)
 #define CHOOSE_BACKEND(name, id) EVALUATOR(name, id)
 
 // Memory management
@@ -152,7 +152,7 @@
   #define PARALLEL_SYNC PARALLEL_SYNC_default
 #endif
 
-// General 
+// General
 
 #if defined(PlusEquals_cuda) || defined(PlusEquals_kokkos) || defined(PlusEquals_omp)
   #define PlusEquals CHOOSE_BACKEND(DEFER(PlusEquals), ACC_ID)
diff --git a/pfsimulator/parflow_lib/background.c b/pfsimulator/parflow_lib/background.c
index 1d048f3d2..54628bbfe 100644
--- a/pfsimulator/parflow_lib/background.c
+++ b/pfsimulator/parflow_lib/background.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routines associated with the Background structure.
diff --git a/pfsimulator/parflow_lib/background.h b/pfsimulator/parflow_lib/background.h
index c7a40ef24..f0dd9db61 100644
--- a/pfsimulator/parflow_lib/background.h
+++ b/pfsimulator/parflow_lib/background.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 * Header info for the Background structure
@@ -69,18 +69,18 @@ typedef struct {
 #define BackgroundNZ(bg)  ((bg)->NZ)
 
 #define BackgroundXLower(bg) \
-  (BackgroundX(bg) + BackgroundIX(bg) * BackgroundDX(bg))
+        (BackgroundX(bg) + BackgroundIX(bg) * BackgroundDX(bg))
 #define BackgroundYLower(bg) \
-  (BackgroundY(bg) + BackgroundIY(bg) * BackgroundDY(bg))
+        (BackgroundY(bg) + BackgroundIY(bg) * BackgroundDY(bg))
 #define BackgroundZLower(bg) \
-  (BackgroundZ(bg) + BackgroundIZ(bg) * BackgroundDZ(bg))
+        (BackgroundZ(bg) + BackgroundIZ(bg) * BackgroundDZ(bg))
 
 #define BackgroundXUpper(bg) \
-  (BackgroundX(bg) + (BackgroundIX(bg) + BackgroundNX(bg)) * BackgroundDX(bg))
+        (BackgroundX(bg) + (BackgroundIX(bg) + BackgroundNX(bg)) * BackgroundDX(bg))
 #define BackgroundYUpper(bg) \
-  (BackgroundY(bg) + (BackgroundIY(bg) + BackgroundNY(bg)) * BackgroundDY(bg))
+        (BackgroundY(bg) + (BackgroundIY(bg) + BackgroundNY(bg)) * BackgroundDY(bg))
 #define BackgroundZUpper(bg) \
-  (BackgroundZ(bg) + (BackgroundIZ(bg) + BackgroundNZ(bg)) * BackgroundDZ(bg))
+        (BackgroundZ(bg) + (BackgroundIZ(bg) + BackgroundNZ(bg)) * BackgroundDZ(bg))
 
 
 #endif
diff --git a/pfsimulator/parflow_lib/bc_lb.c b/pfsimulator/parflow_lib/bc_lb.c
index 675e4c4ee..bc6176179 100644
--- a/pfsimulator/parflow_lib/bc_lb.c
+++ b/pfsimulator/parflow_lib/bc_lb.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
diff --git a/pfsimulator/parflow_lib/bc_pressure.c b/pfsimulator/parflow_lib/bc_pressure.c
index 81bca3df1..fbd4f2fc0 100644
--- a/pfsimulator/parflow_lib/bc_pressure.c
+++ b/pfsimulator/parflow_lib/bc_pressure.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
diff --git a/pfsimulator/parflow_lib/bc_pressure.h b/pfsimulator/parflow_lib/bc_pressure.h
index 74abb7293..729478886 100644
--- a/pfsimulator/parflow_lib/bc_pressure.h
+++ b/pfsimulator/parflow_lib/bc_pressure.h
@@ -1,11 +1,30 @@
-/*BHEADER*********************************************************************
-* (c) 1995   The Regents of the University of California
+/*BHEADER**********************************************************************
 *
-* See the file COPYRIGHT_and_DISCLAIMER for a complete copyright
-* notice, contact person, and disclaimer.
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
 *
-* $Revision: 1.1.1.1 $
-*********************************************************************EHEADER*/
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef _BC_PRESSURE_HEADER
 #define _BC_PRESSURE_HEADER
@@ -75,52 +94,52 @@
  * These contain information for all interval steps to be read later
  * These structs will be local to the BCPressurePackage source file only
  *----------------------------------------------------------------*/
-#define BC_TYPE_TABLE            \
-  BC_TYPE(DirEquilRefPatch, {    \
-    int reference_solid;         \
-    int reference_patch;         \
-    double *values;              \
-    double **value_at_interface; \
-  })                             \
-  BC_TYPE(DirEquilPLinear, {     \
-    double *xlower;              \
-    double *ylower;              \
-    double *xupper;              \
-    double *yupper;              \
-    int    *num_points;          \
-    double **points;             \
-    double **values;             \
-    double **value_at_interface; \
-  })                             \
-  BC_TYPE(FluxConst, {           \
-    double *values;              \
-  })                             \
-  BC_TYPE(FluxVolumetric, {      \
-    double *values;              \
-  })                             \
-  BC_TYPE(PressureFile, {        \
-    char **filenames;            \
-  })                             \
-  BC_TYPE(FluxFile, {            \
-    char **filenames;            \
-  })                             \
-  BC_TYPE(ExactSolution, {       \
-    int function_type;           \
-  })                             \
-  BC_TYPE(OverlandFlow, {        \
-    double *values;              \
-  })                             \
-  BC_TYPE(OverlandFlowPFB, {     \
-    char **filenames;            \
-  })                             \
-  BC_TYPE(SeepageFace, {         \
-    double *values;              \
-  })                             \
-  BC_TYPE(OverlandKinematic, {   \
-    double *values;              \
-  })                             \
-  BC_TYPE(OverlandDiffusive, {   \
-    double *values;              \
+#define BC_TYPE_TABLE                  \
+        BC_TYPE(DirEquilRefPatch, {    \
+    int reference_solid;               \
+    int reference_patch;               \
+    double *values;                    \
+    double **value_at_interface;       \
+  })                                   \
+        BC_TYPE(DirEquilPLinear, {     \
+    double *xlower;                    \
+    double *ylower;                    \
+    double *xupper;                    \
+    double *yupper;                    \
+    int    *num_points;                \
+    double **points;                   \
+    double **values;                   \
+    double **value_at_interface;       \
+  })                                   \
+        BC_TYPE(FluxConst, {           \
+    double *values;                    \
+  })                                   \
+        BC_TYPE(FluxVolumetric, {      \
+    double *values;                    \
+  })                                   \
+        BC_TYPE(PressureFile, {        \
+    char **filenames;                  \
+  })                                   \
+        BC_TYPE(FluxFile, {            \
+    char **filenames;                  \
+  })                                   \
+        BC_TYPE(ExactSolution, {       \
+    int function_type;                 \
+  })                                   \
+        BC_TYPE(OverlandFlow, {        \
+    double *values;                    \
+  })                                   \
+        BC_TYPE(OverlandFlowPFB, {     \
+    char **filenames;                  \
+  })                                   \
+        BC_TYPE(SeepageFace, {         \
+    double *values;                    \
+  })                                   \
+        BC_TYPE(OverlandKinematic, {   \
+    double *values;                    \
+  })                                   \
+        BC_TYPE(OverlandDiffusive, {   \
+    double *values;                    \
   })
 
 
@@ -132,7 +151,7 @@
  * @param var Variable name for newly declared and allocated TypeStruct pointer
  */
 #define NewTypeStruct(type, var) \
-  Type ## type * var = ctalloc(Type ## type, 1)
+        Type ## type * var = ctalloc(Type ## type, 1)
 
 /**
  * @name TypeStruct Setter
@@ -143,7 +162,7 @@
  * @param i Patch index
  */
 #define StoreTypeStruct(public_xtra, var, i) \
-  (public_xtra)->data[(i)] = (void*)(var);
+        (public_xtra)->data[(i)] = (void*)(var);
 
 /**
  * @name TypeStruct Accessor
@@ -155,14 +174,14 @@
  * @param i Patch index
  */
 #define GetTypeStruct(type, var, public_xtra, i) \
-  Type ## type * var = (Type ## type*)(public_xtra->data[i])
+        Type ## type * var = (Type ## type*)(public_xtra->data[i])
 
 
 // MCB: These two macros aren't really necessary but they do make the code cleaner
 #define ForEachPatch(num_patches, i) \
-  for (i = 0; i < num_patches; i++)
+        for (i = 0; i < num_patches; i++)
 #define ForEachInterval(interval_division, interval_number) \
-  for (interval_number = 0; interval_number < interval_division; interval_number++)
+        for (interval_number = 0; interval_number < interval_division; interval_number++)
 
 
 /*----------------------------------------------------------------
@@ -172,52 +191,52 @@
  *       the type struct definitions, with one less reference level.
  *       Could use C++ std::remove_pointer to only need one table def.
  *----------------------------------------------------------------*/
-#define BC_INTERVAL_TYPE_TABLE   \
-  BC_TYPE(DirEquilRefPatch, {    \
-    int reference_solid;         \
-    int reference_patch;         \
-    double value;                \
-    double *value_at_interfaces; \
-  })                             \
-  BC_TYPE(DirEquilPLinear, {     \
-    double xlower;               \
-    double ylower;               \
-    double xupper;               \
-    double yupper;               \
-    int num_points;              \
-    double *points;              \
-    double *values;              \
-    double *value_at_interfaces; \
-  })                             \
-  BC_TYPE(FluxConst, {           \
-    double value;                \
-  })                             \
-  BC_TYPE(FluxVolumetric, {      \
-    double value;                \
-  })                             \
-  BC_TYPE(PressureFile, {        \
-    char *filename;              \
-  })                             \
-  BC_TYPE(FluxFile, {            \
-    char *filename;              \
-  })                             \
-  BC_TYPE(ExactSolution, {       \
-    int function_type;           \
-  })                             \
-  BC_TYPE(OverlandFlow, {        \
-    double value;                \
-  })                             \
-  BC_TYPE(OverlandFlowPFB, {     \
-    char *filename;              \
-  })                             \
-  BC_TYPE(SeepageFace, {         \
-    double value;                \
-  })                             \
-  BC_TYPE(OverlandKinematic, {   \
-    double value;                \
-  })                             \
-  BC_TYPE(OverlandDiffusive, {   \
-    double value;                \
+#define BC_INTERVAL_TYPE_TABLE         \
+        BC_TYPE(DirEquilRefPatch, {    \
+    int reference_solid;               \
+    int reference_patch;               \
+    double value;                      \
+    double *value_at_interfaces;       \
+  })                                   \
+        BC_TYPE(DirEquilPLinear, {     \
+    double xlower;                     \
+    double ylower;                     \
+    double xupper;                     \
+    double yupper;                     \
+    int num_points;                    \
+    double *points;                    \
+    double *values;                    \
+    double *value_at_interfaces;       \
+  })                                   \
+        BC_TYPE(FluxConst, {           \
+    double value;                      \
+  })                                   \
+        BC_TYPE(FluxVolumetric, {      \
+    double value;                      \
+  })                                   \
+        BC_TYPE(PressureFile, {        \
+    char *filename;                    \
+  })                                   \
+        BC_TYPE(FluxFile, {            \
+    char *filename;                    \
+  })                                   \
+        BC_TYPE(ExactSolution, {       \
+    int function_type;                 \
+  })                                   \
+        BC_TYPE(OverlandFlow, {        \
+    double value;                      \
+  })                                   \
+        BC_TYPE(OverlandFlowPFB, {     \
+    char *filename;                    \
+  })                                   \
+        BC_TYPE(SeepageFace, {         \
+    double value;                      \
+  })                                   \
+        BC_TYPE(OverlandKinematic, {   \
+    double value;                      \
+  })                                   \
+        BC_TYPE(OverlandDiffusive, {   \
+    double value;                      \
   })
 
 
@@ -235,7 +254,7 @@ BC_INTERVAL_TYPE_TABLE
  * @param varname Name of the variable to declare and allocate
  */
 #define NewBCPressureTypeStruct(type, varname) \
-  BCPressureType ## type * varname = ctalloc(BCPressureType ## type, 1)
+        BCPressureType ## type * varname = ctalloc(BCPressureType ## type, 1)
 
 /**
  * @name BCPressureType Accessor
@@ -247,11 +266,11 @@ BC_INTERVAL_TYPE_TABLE
  * @param ipatch Patch index
  * @param interval_number Interval index
  */
-#define GetBCPressureTypeStruct(type, varname, bc_pressure_data, ipatch, interval_number) \
-  BCPressureType ## type * varname                                                        \
-    = (BCPressureType ## type*)BCPressureDataIntervalValue(bc_pressure_data,              \
-                                                           (ipatch), (interval_number));  \
-  PF_UNUSED(varname)
+#define GetBCPressureTypeStruct(type, varname, bc_pressure_data, ipatch, interval_number)        \
+        BCPressureType ## type * varname                                                         \
+          = (BCPressureType ## type*)BCPressureDataIntervalValue(bc_pressure_data,               \
+                                                                 (ipatch), (interval_number));   \
+        PF_UNUSED(varname)
 
 /*----------------------------------------------------------------
  * BCPressure Data structure
@@ -277,120 +296,120 @@ typedef struct {
  * @MCB: With the new macro system these don't make as much sense, replace/deprecate?
  *--------------------------------------------------------------------------*/
 #define DirEquilRefPatchValue(patch) \
-  ((patch)->value)
+        ((patch)->value)
 
 #define DirEquilRefPatchRefSolid(patch) \
-  ((patch)->reference_solid)
+        ((patch)->reference_solid)
 #define DirEquilRefPatchRefPatch(patch) \
-  ((patch)->reference_patch)
+        ((patch)->reference_patch)
 
 #define DirEquilRefPatchValueAtInterfaces(patch) \
-  ((patch)->value_at_interfaces)
+        ((patch)->value_at_interfaces)
 #define DirEquilRefPatchValueAtInterface(patch, i) \
-  ((patch)->value_at_interfaces[i - 1])
+        ((patch)->value_at_interfaces[i - 1])
 /*--------------------------------------------------------------------------*/
 #define DirEquilPLinearXLower(patch) \
-  ((patch)->xlower)
+        ((patch)->xlower)
 
 #define DirEquilPLinearYLower(patch) \
-  ((patch)->ylower)
+        ((patch)->ylower)
 
 #define DirEquilPLinearXUpper(patch) \
-  ((patch)->xupper)
+        ((patch)->xupper)
 
 #define DirEquilPLinearYUpper(patch) \
-  ((patch)->yupper)
+        ((patch)->yupper)
 
 #define DirEquilPLinearNumPoints(patch) \
-  ((patch)->num_points)
+        ((patch)->num_points)
 
 #define DirEquilPLinearPoints(patch) \
-  ((patch)->points)
+        ((patch)->points)
 #define DirEquilPLinearPoint(patch, i) \
-  ((patch)->points[i])
+        ((patch)->points[i])
 
 #define DirEquilPLinearValues(patch) \
-  ((patch)->values)
+        ((patch)->values)
 #define DirEquilPLinearValue(patch, i) \
-  ((patch)->values[i])
+        ((patch)->values[i])
 
 #define DirEquilPLinearValueAtInterfaces(patch) \
-  ((patch)->value_at_interfaces)
+        ((patch)->value_at_interfaces)
 #define DirEquilPLinearValueAtInterface(patch, i) \
-  ((patch)->value_at_interfaces[i - 1])
+        ((patch)->value_at_interfaces[i - 1])
 /*--------------------------------------------------------------------------*/
 #define FluxConstValue(patch) \
-  ((patch)->value)
+        ((patch)->value)
 /*--------------------------------------------------------------------------*/
 #define FluxVolumetricValue(patch) \
-  ((patch)->value)
+        ((patch)->value)
 
 /*--------------------------------------------------------------------------*/
 #define PressureFileName(patch) \
-  ((patch)->filename)
+        ((patch)->filename)
 
 /*--------------------------------------------------------------------------*/
 #define FluxFileName(patch) \
-  ((patch)->filename)
+        ((patch)->filename)
 
 /*--------------------------------------------------------------------------*/
 #define ExactSolutionFunctionType(patch) \
-  ((patch)->function_type)
+        ((patch)->function_type)
 
 /*--------------------------------------------------------------------------*/
 #define OverlandFlowValue(patch) \
-  ((patch)->value)
+        ((patch)->value)
 /*--------------------------------------------------------------------------*/
 #define OverlandFlowPFBFileName(patch) \
-  ((patch)->filename)
+        ((patch)->filename)
 /*--------------------------------------------------------------------------*/
 #define SeepageFaceValue(patch) \
-  ((patch)->value)
+        ((patch)->value)
 
 /*--------------------------------------------------------------------------*/
   #define OverlandKinematicValue(patch) \
-  ((patch)->value)
+          ((patch)->value)
 /*--------------------------------------------------------------------------*/
     #define OverlandDiffusiveValue(patch) \
-  ((patch)->value)
+            ((patch)->value)
 /*--------------------------------------------------------------------------
  * Accessor macros: BCPressureData
  *--------------------------------------------------------------------------*/
 #define BCPressureDataNumPhases(bc_pressure_data) \
-  ((bc_pressure_data)->num_phases)
+        ((bc_pressure_data)->num_phases)
 
 #define BCPressureDataNumPatches(bc_pressure_data) \
-  ((bc_pressure_data)->num_patches)
+        ((bc_pressure_data)->num_patches)
 
 #define BCPressureDataTypes(bc_pressure_data) \
-  ((bc_pressure_data)->types)
+        ((bc_pressure_data)->types)
 #define BCPressureDataType(bc_pressure_data, i) \
-  ((bc_pressure_data)->types[i])
+        ((bc_pressure_data)->types[i])
 
 #define BCPressureDataCycleNumbers(bc_pressure_data) \
-  ((bc_pressure_data)->cycle_numbers)
+        ((bc_pressure_data)->cycle_numbers)
 #define BCPressureDataCycleNumber(bc_pressure_data, i) \
-  ((bc_pressure_data)->cycle_numbers[i])
+        ((bc_pressure_data)->cycle_numbers[i])
 
 #define BCPressureDataPatchIndexes(bc_pressure_data) \
-  ((bc_pressure_data)->patch_indexes)
+        ((bc_pressure_data)->patch_indexes)
 #define BCPressureDataPatchIndex(bc_pressure_data, i) \
-  ((bc_pressure_data)->patch_indexes[i])
+        ((bc_pressure_data)->patch_indexes[i])
 
 #define BCPressureDataBCTypes(bc_pressure_data) \
-  ((bc_pressure_data)->bc_types)
+        ((bc_pressure_data)->bc_types)
 #define BCPressureDataBCType(bc_pressure_data, i) \
-  ((bc_pressure_data)->bc_types[i])
+        ((bc_pressure_data)->bc_types[i])
 
 #define BCPressureDataValues(bc_pressure_data) \
-  ((bc_pressure_data)->values)
+        ((bc_pressure_data)->values)
 #define BCPressureDataIntervalValues(bc_pressure_data, i) \
-  ((bc_pressure_data)->values[i])
+        ((bc_pressure_data)->values[i])
 #define BCPressureDataIntervalValue(bc_pressure_data, i, interval_number) \
-  (((bc_pressure_data)->values[i])[interval_number])
+        (((bc_pressure_data)->values[i])[interval_number])
 
 #define BCPressureDataTimeCycleData(bc_pressure_data) \
-  ((bc_pressure_data)->time_cycle_data)
+        ((bc_pressure_data)->time_cycle_data)
 
 /** @} */
 
diff --git a/pfsimulator/parflow_lib/bc_pressure_package.c b/pfsimulator/parflow_lib/bc_pressure_package.c
index f0294ac3b..be21030e2 100644
--- a/pfsimulator/parflow_lib/bc_pressure_package.c
+++ b/pfsimulator/parflow_lib/bc_pressure_package.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -86,7 +86,7 @@ void         BCPressurePackage(
   int num_patches;
   int i;
   int cycle_length, cycle_number, interval_division,
-    interval_number;
+      interval_number;
 
   /* Allocate the bc data */
   BCPressureDataNumPhases(bc_pressure_data) = (public_xtra->num_phases);
@@ -139,10 +139,10 @@ void         BCPressurePackage(
       BCPressureDataIntervalValues(bc_pressure_data, i) = ctalloc(void *, interval_division);
       for (interval_number = 0; interval_number < interval_division; interval_number++)
       {
-	switch ((public_xtra)->input_types[(i)])
+        switch ((public_xtra)->input_types[(i)])
         {
 #if 0 /* Do not uncomment this block, it is example code */
-	  case MyNewPatchType:
+          case MyNewPatchType:
           {
             /* Allocate the struct for an interval */
             NewBCPressureTypeStruct(MyNewPatchType, interval_data);
@@ -160,8 +160,8 @@ void         BCPressurePackage(
             /* Save the pointer into bc_pressure_data */
             BCPressureDataIntervalValue(bc_pressure_data, i, interval_number)
               = (void*)interval_data;
-	    
-	    break;
+
+            break;
           } /* End MyNewPatchType */
 #endif // #if 0
 
@@ -200,7 +200,7 @@ void         BCPressurePackage(
             }
 
             BCPressureDataIntervalValue(bc_pressure_data, i, interval_number) = (void*)interval_data;
-	    break;
+            break;
           } /* End DirEquilRefPatch */
 
           /* Setup a piecewise linear pressure condition structure */
@@ -250,8 +250,8 @@ void         BCPressurePackage(
             }
 
             BCPressureDataIntervalValue(bc_pressure_data, i, interval_number) = (void*)interval_data;
-	    
-	    break;
+
+            break;
           } /* End DirEquilPLinear */
 
           /* Setup a constant flux condition structure */
@@ -269,7 +269,7 @@ void         BCPressurePackage(
             BCPressureDataIntervalValue(bc_pressure_data, i, interval_number)
               = (void*)interval_data;
 
-	    break;
+            break;
           } /* End FluxConst */
 
           /* Setup a volumetric flux condition structure */
@@ -286,7 +286,7 @@ void         BCPressurePackage(
 
             BCPressureDataIntervalValue(bc_pressure_data, i, interval_number)
               = (void*)interval_data;
-	    break;
+            break;
           } /* End FluxVolumetric */
 
           /* Setup a file defined pressure condition structure */
@@ -306,7 +306,7 @@ void         BCPressurePackage(
 
             BCPressureDataIntervalValue(bc_pressure_data, i, interval_number)
               = (void*)interval_data;
-	    break;
+            break;
           } /* End PressureFile */
 
           /* Setup a file defined flux condition structure */
@@ -326,8 +326,8 @@ void         BCPressurePackage(
 
             BCPressureDataIntervalValue(bc_pressure_data, i, interval_number)
               = (void*)interval_data;
-	    
-	    break;
+
+            break;
           } /* End FluxFile */
 
           /* Setup a Dir. pressure MATH problem condition */
@@ -344,8 +344,8 @@ void         BCPressurePackage(
 
             BCPressureDataIntervalValue(bc_pressure_data, i, interval_number)
               = (void*)interval_data;
-	    
-	    break;
+
+            break;
           } /* End ExactSolution */
 
           /*//sk  Setup a overland flow condition structure */
@@ -362,7 +362,7 @@ void         BCPressurePackage(
 
             BCPressureDataIntervalValue(bc_pressure_data, i, interval_number)
               = (void*)interval_data;
-	    break;
+            break;
           } /* End OverlandFlow */
 
           /* Setup a file defined flux condition structure for overland flow BC*/
@@ -382,8 +382,8 @@ void         BCPressurePackage(
 
             BCPressureDataIntervalValue(bc_pressure_data, i, interval_number)
               = (void*)interval_data;
-	    
-	    break;
+
+            break;
           } /* End OverlandFlowPFB */
 
           /* Set up a seepage face condition structure */
@@ -400,10 +400,9 @@ void         BCPressurePackage(
 
             BCPressureDataIntervalValue(bc_pressure_data, i, interval_number)
               = (void*)interval_data;
-	    break;
-	    
+            break;
           } /* End SeepageFace */
-	  
+
           /* Set up overland OverlandKinematic condition structure */
           case OverlandKinematic:
           {
@@ -419,11 +418,11 @@ void         BCPressurePackage(
             BCPressureDataIntervalValue(bc_pressure_data, i, interval_number)
               = (void*)interval_data;
 
-	    break;
+            break;
           } /* End OverlandKinematic */
-	  
+
           /* Set up overland OverlandDiffusive condition structure */
-	  case OverlandDiffusive:
+          case OverlandDiffusive:
           {
             NewBCPressureTypeStruct(OverlandDiffusive, interval_data);
 
@@ -437,14 +436,13 @@ void         BCPressurePackage(
             BCPressureDataIntervalValue(bc_pressure_data, i, interval_number)
               = (void*)interval_data;
 
-	    break;
-	  } /* End OverlandDiffusive */
+            break;
+          } /* End OverlandDiffusive */
 
-	  default:
-	  {
-	    PARFLOW_ERROR("Invalid BC input type");
-	  }
-	  
+          default:
+          {
+            PARFLOW_ERROR("Invalid BC input type");
+          }
         } /* End switch BC type  */
       } /* End for interval */
     } /* End for patch */
@@ -572,10 +570,10 @@ PFModule  *BCPressurePackageNewPublicXtra(
         NA_NameToIndex(GlobalsGeometries[domain_index]->patches,
                        patch_name);
 
-      if(public_xtra->patch_indexes[i] < 0)
+      if (public_xtra->patch_indexes[i] < 0)
       {
-	amps_Printf("Invalid patch name <%s>\n", patch_name);
-	NA_InputError(GlobalsGeometries[domain_index]->patches, patch_name, "");
+        amps_Printf("Invalid patch name <%s>\n", patch_name);
+        NA_InputError(GlobalsGeometries[domain_index]->patches, patch_name, "");
       }
 
       sprintf(key, "Patch.%s.BCPressure.Type", patch_name);
@@ -603,41 +601,42 @@ PFModule  *BCPressurePackageNewPublicXtra(
 
       switch ((public_xtra)->input_types[(i)])
       {
-	
 #if 0 /* Do not undef this block, it is example code not for actual use */
-        /* Example flow for setting up TypeStruct for boundary conditions */
-        switch yNewPatchType:
-        {
-          /* Allocate the struct, second parameter is whatever variable name you wish to use in this scope */
-          NewTypeStruct(MyNewPatchType, data);
-
-          /* Allocate struct data if necessary */
-          (data->values) = ctalloc(double, interval_division);
-
-          /* Populate the data with something, e.g. values from a file */
-          ForEachInterval(interval_division, interval_number)
-          {
-            sprintf(key, "Patch.%s.BCPressure.%s.Value",
-                    patch_name,
-                    NA_IndexToName(
-                                   GlobalsIntervalNames[global_cycle],
-                                   interval_number));
-
-            data->values[interval_number] = GetDouble(key);
-          }
-
-          /* Store the allocated and populated Type struct into public_xtra */
-          StoreTypeStruct(public_xtra, data, i);
-	  break;
-        } /* End Example */
+      /* Example flow for setting up TypeStruct for boundary conditions */
+      /*
+       * switch yNewPatchType:
+       * {
+       * // Allocate the struct, second parameter is whatever variable name you wish to use in this scope
+       * NewTypeStruct(MyNewPatchType, data);
+       *
+       * // Allocate struct data if necessary
+       * (data->values) = ctalloc(double, interval_division);
+       *
+       * // Populate the data with something, e.g. values from a file
+       * ForEachInterval(interval_division, interval_number)
+       * {
+       *  sprintf(key, "Patch.%s.BCPressure.%s.Value",
+       *          patch_name,
+       *          NA_IndexToName(
+       *                         GlobalsIntervalNames[global_cycle],
+       *                         interval_number));
+       *
+       *  data->values[interval_number] = GetDouble(key);
+       * }
+       *
+       * // Store the allocated and populated Type struct into public_xtra
+       * StoreTypeStruct(public_xtra, data, i);
+       * break;
+       * } // End Example
+       */
 #endif // #if 0
 
         case DirEquilRefPatch:
         {
           int size;
-	  
+
           NewTypeStruct(DirEquilRefPatch, data);
-	  
+
           (data->values) = ctalloc(double,
                                    interval_division);
           (data->value_at_interface) = ctalloc(double *,
@@ -659,8 +658,8 @@ PFModule  *BCPressurePackageNewPublicXtra(
 
           data->reference_patch =
             NA_NameToIndexExitOnError(GeomSolidPatches(
-						       GlobalsGeometries[data->reference_solid]),
-				      switch_name, key);
+                                                       GlobalsGeometries[data->reference_solid]),
+                                      switch_name, key);
 
           ForEachInterval(interval_division, interval_number)
           {
@@ -697,16 +696,16 @@ PFModule  *BCPressurePackageNewPublicXtra(
               (data->value_at_interface[interval_number]) = NULL;
             }
           }
-	  
+
           StoreTypeStruct(public_xtra, data, i);
-	  
-	  break;
+
+          break;
         } /* End DirEquilRefPatch */
-	
+
         case DirEquilPLinear:
-	{
-	  int k;
-	  int num_points;
+        {
+          int k;
+          int num_points;
           int size;
 
           NewTypeStruct(DirEquilPLinear, data);
@@ -793,15 +792,15 @@ PFModule  *BCPressurePackageNewPublicXtra(
           }
 
           StoreTypeStruct(public_xtra, data, i);
-	  break;
+          break;
         } /* End DirEquilPLinear */
-			    
+
         case FluxConst:
         {
           NewTypeStruct(FluxConst, data);
-	  
+
           (data->values) = ctalloc(double, interval_division);
-	  
+
           ForEachInterval(interval_division, interval_number)
           {
             sprintf(key, "Patch.%s.BCPressure.%s.Value",
@@ -813,11 +812,11 @@ PFModule  *BCPressurePackageNewPublicXtra(
           }
 
           StoreTypeStruct(public_xtra, data, i);
-	  
-	  break;
+
+          break;
         } /* End FluxConst */
 
-	case FluxVolumetric:
+        case FluxVolumetric:
         {
           NewTypeStruct(FluxVolumetric, data);
 
@@ -834,8 +833,8 @@ PFModule  *BCPressurePackageNewPublicXtra(
           }
 
           StoreTypeStruct(public_xtra, data, i);
-	  
-	  break;
+
+          break;
         } /* End FluxVolumetric */
 
         case PressureFile:
@@ -856,10 +855,10 @@ PFModule  *BCPressurePackageNewPublicXtra(
 
           StoreTypeStruct(public_xtra, data, i);
 
-	  break;
+          break;
         } /* End PressureFile */
 
-	case FluxFile:
+        case FluxFile:
         {
           NewTypeStruct(FluxFile, data);
 
@@ -876,8 +875,8 @@ PFModule  *BCPressurePackageNewPublicXtra(
           }
 
           StoreTypeStruct(public_xtra, data, i);
-	  
-	  break;
+
+          break;
         } /* End FluxFile */
 
         case ExactSolution:
@@ -893,7 +892,7 @@ PFModule  *BCPressurePackageNewPublicXtra(
             switch_name = GetString(key);
 
             data->function_type = NA_NameToIndexExitOnError(function_na,
-							    switch_name, key);
+                                                            switch_name, key);
 
             // MCB: This is overwriting the data struct inside the for loop
             // Also structured this way in master branch without changes
@@ -901,36 +900,36 @@ PFModule  *BCPressurePackageNewPublicXtra(
             StoreTypeStruct(public_xtra, data, i);
           }
 
-	  break;
+          break;
         } /* End ExactSolution */
 
         case OverlandFlow:
         {
           NewTypeStruct(OverlandFlow, data);
-	  
+
           (data->values) = ctalloc(double, interval_division);
-	  
+
           ForEachInterval(interval_division, interval_number)
           {
             sprintf(key, "Patch.%s.BCPressure.%s.Value",
                     patch_name,
                     NA_IndexToName(GlobalsIntervalNames[global_cycle],
                                    interval_number));
-	    
+
             data->values[interval_number] = GetDouble(key);
           }
-	  
+
           StoreTypeStruct(public_xtra, data, i);
-	  
-	  break;
+
+          break;
         } /* End OverlandFlow */
 
-	case OverlandFlowPFB:
+        case OverlandFlowPFB:
         {
           NewTypeStruct(OverlandFlowPFB, data);
-	  
+
           (data->filenames) = ctalloc(char *, interval_division);
-	  
+
           ForEachInterval(interval_division, interval_number)
           {
             sprintf(key, "Patch.%s.BCPressure.%s.FileName",
@@ -940,14 +939,14 @@ PFModule  *BCPressurePackageNewPublicXtra(
 
             data->filenames[interval_number] = GetString(key);
           }
-	  
+
           StoreTypeStruct(public_xtra, data, i);
 
-	  break;
+          break;
         } /* End OverlandFlowPFB */
 
-	case SeepageFace:
-	{
+        case SeepageFace:
+        {
           /* Constant "rainfall" rate value on patch */
           NewTypeStruct(SeepageFace, data);
 
@@ -966,8 +965,8 @@ PFModule  *BCPressurePackageNewPublicXtra(
             data->values[interval_number] = GetDouble(key);
           }
           StoreTypeStruct(public_xtra, data, i);
-	  
-	  break;
+
+          break;
         } /* End SeepageFace */
 
         case OverlandKinematic:
@@ -987,7 +986,7 @@ PFModule  *BCPressurePackageNewPublicXtra(
           }
           StoreTypeStruct(public_xtra, data, i);
 
-	  break;
+          break;
         } /* End OverlandKinematic */
 
         case OverlandDiffusive:
@@ -1006,10 +1005,9 @@ PFModule  *BCPressurePackageNewPublicXtra(
             data->values[interval_number] = GetDouble(key);
           }
           StoreTypeStruct(public_xtra, data, i);
-	  
-	  break;
+
+          break;
         } /* End OverlandDiffusive */
-	
       } /* End switch types */
     } /* End for patches */
   } /* if patches */
@@ -1045,8 +1043,8 @@ void  BCPressurePackageFreePublicXtra()
       for (i = 0; i < num_patches; i++)
       {
         interval_division = (public_xtra->interval_divisions[(public_xtra->cycle_numbers[i])]);
-	switch ((public_xtra)->input_types[(i)])
-	{
+        switch ((public_xtra)->input_types[(i)])
+        {
           case DirEquilRefPatch:
           {
             GetTypeStruct(DirEquilRefPatch, data, public_xtra, i);
@@ -1060,7 +1058,7 @@ void  BCPressurePackageFreePublicXtra()
             tfree((data->values));
 
             tfree(data);
-	    break;
+            break;
           }
 
           case DirEquilPLinear:
@@ -1088,7 +1086,7 @@ void  BCPressurePackageFreePublicXtra()
             tfree((data->xlower));
 
             tfree(data);
-	    break;
+            break;
           }
 
           case FluxConst:
@@ -1096,7 +1094,7 @@ void  BCPressurePackageFreePublicXtra()
             GetTypeStruct(FluxConst, data, public_xtra, i);
             tfree((data->values));
             tfree(data);
-	    break;
+            break;
           }
 
           case FluxVolumetric:
@@ -1104,7 +1102,7 @@ void  BCPressurePackageFreePublicXtra()
             GetTypeStruct(FluxVolumetric, data, public_xtra, i);
             tfree((data->values));
             tfree(data);
-	    break;
+            break;
           }
 
           case PressureFile:
@@ -1112,7 +1110,7 @@ void  BCPressurePackageFreePublicXtra()
             GetTypeStruct(PressureFile, data, public_xtra, i);
             tfree((data->filenames));
             tfree(data);
-	    break;
+            break;
           }
 
           case FluxFile:
@@ -1120,14 +1118,14 @@ void  BCPressurePackageFreePublicXtra()
             GetTypeStruct(FluxFile, data, public_xtra, i);
             tfree((data->filenames));
             tfree(data);
-	    break;
+            break;
           }
 
           case ExactSolution:
           {
             GetTypeStruct(ExactSolution, data, public_xtra, i);
             tfree(data);
-	    break;
+            break;
           }
 
           //sk
@@ -1136,7 +1134,7 @@ void  BCPressurePackageFreePublicXtra()
             GetTypeStruct(OverlandFlow, data, public_xtra, i);
             tfree((data->values));
             tfree(data);
-	    break;
+            break;
           }
 
           //RMM
@@ -1145,7 +1143,7 @@ void  BCPressurePackageFreePublicXtra()
             GetTypeStruct(OverlandFlowPFB, data, public_xtra, i);
             tfree((data->filenames));
             tfree(data);
-	    break;
+            break;
           }
 
           case SeepageFace:
@@ -1153,7 +1151,7 @@ void  BCPressurePackageFreePublicXtra()
             GetTypeStruct(SeepageFace, data, public_xtra, i);
             tfree(data->values);
             tfree(data);
-	    break;
+            break;
           };
 
           case OverlandKinematic:
@@ -1161,7 +1159,7 @@ void  BCPressurePackageFreePublicXtra()
             GetTypeStruct(OverlandKinematic, data, public_xtra, i);
             tfree(data->values);
             tfree(data);
-	    break;
+            break;
           }
 
           case OverlandDiffusive:
@@ -1169,14 +1167,13 @@ void  BCPressurePackageFreePublicXtra()
             GetTypeStruct(OverlandDiffusive, data, public_xtra, i);
             tfree(data->values);
             tfree(data);
-	    break;
+            break;
           }
 
-	  default:
-	  {
-	    PARFLOW_ERROR("Invalid BC input type");
-	  }
-	  
+          default:
+          {
+            PARFLOW_ERROR("Invalid BC input type");
+          }
         } /* End switch type */
       } /* End for patch */
 
diff --git a/pfsimulator/parflow_lib/calc_elevations.c b/pfsimulator/parflow_lib/calc_elevations.c
index b463b34b5..02c690f12 100644
--- a/pfsimulator/parflow_lib/calc_elevations.c
+++ b/pfsimulator/parflow_lib/calc_elevations.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *****************************************************************************/
diff --git a/pfsimulator/parflow_lib/cghs.c b/pfsimulator/parflow_lib/cghs.c
index 27159826c..f5db0bba0 100644
--- a/pfsimulator/parflow_lib/cghs.c
+++ b/pfsimulator/parflow_lib/cghs.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
diff --git a/pfsimulator/parflow_lib/char_vector.c b/pfsimulator/parflow_lib/char_vector.c
index ba4b22a52..8b5c17474 100644
--- a/pfsimulator/parflow_lib/char_vector.c
+++ b/pfsimulator/parflow_lib/char_vector.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routines for manipulating charvector structures.
diff --git a/pfsimulator/parflow_lib/char_vector.h b/pfsimulator/parflow_lib/char_vector.h
index 1e5a3c30b..35ba4b01b 100644
--- a/pfsimulator/parflow_lib/char_vector.h
+++ b/pfsimulator/parflow_lib/char_vector.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Header info for the CharVector data structures
@@ -79,7 +79,7 @@ typedef struct _CharVector {
 
 #define SubcharvectorData(subcharvector) ((subcharvector)->data)
 #define SubcharvectorCompData(subcharvector, ci) \
-  (((subcharvector)->data) + ((subcharvector)->data_index[ci]))
+        (((subcharvector)->data) + ((subcharvector)->data_index[ci]))
 
 #define SubcharvectorDataSpace(subcharvector)  ((subcharvector)->data_space)
 
@@ -93,16 +93,16 @@ typedef struct _CharVector {
 
 #define SubcharvectorNC(subcharvector)   ((subcharvector)->nc)
 
-#define SubcharvectorEltIndex(subcharvector, x, y, z) \
-  (((x) - SubcharvectorIX(subcharvector)) +           \
-   (((y) - SubcharvectorIY(subcharvector)) +          \
-    (((z) - SubcharvectorIZ(subcharvector))) *        \
-    SubcharvectorNY(subcharvector)) *                 \
-   SubcharvectorNX(subcharvector))
+#define SubcharvectorEltIndex(subcharvector, x, y, z)       \
+        (((x) - SubcharvectorIX(subcharvector)) +           \
+         (((y) - SubcharvectorIY(subcharvector)) +          \
+          (((z) - SubcharvectorIZ(subcharvector))) *        \
+          SubcharvectorNY(subcharvector)) *                 \
+         SubcharvectorNX(subcharvector))
 
-#define SubcharvectorElt(subcharvector, ci, x, y, z) \
-  (SubcharvectorCompData(subcharvector, ci) +        \
-   SubcharvectorEltIndex(subcharvector, x, y, z))
+#define SubcharvectorElt(subcharvector, ci, x, y, z)       \
+        (SubcharvectorCompData(subcharvector, ci) +        \
+         SubcharvectorEltIndex(subcharvector, x, y, z))
 
 /*--------------------------------------------------------------------------
  * Accessor functions for the CharVector structure
diff --git a/pfsimulator/parflow_lib/chebyshev.c b/pfsimulator/parflow_lib/chebyshev.c
index 9be5d6654..4e6d9147f 100644
--- a/pfsimulator/parflow_lib/chebyshev.c
+++ b/pfsimulator/parflow_lib/chebyshev.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
diff --git a/pfsimulator/parflow_lib/clustering.c b/pfsimulator/parflow_lib/clustering.c
index 32d8afe3a..0b56f3d2f 100644
--- a/pfsimulator/parflow_lib/clustering.c
+++ b/pfsimulator/parflow_lib/clustering.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2019, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 #include "clustering.h"
@@ -107,6 +107,7 @@ void HistogramBoxAddTags(HistogramBox *histogram_box, int dim, int global_index,
 HistogramBox* NewHistogramBox(Box *box)
 {
   HistogramBox* histogram_box = talloc(HistogramBox, 1);
+
   memset(histogram_box, 0, sizeof(HistogramBox));
 
   BoxCopy(&(histogram_box->box), box);
@@ -839,7 +840,7 @@ void ComputePatchBoxes(GrGeomSolid *geom_solid, int patch)
       {
         int ip = SubvectorEltIndex(d_sub, i, j, k);
         int this_face_tag = 1 << PV_f;
-        
+
         DoubleTags v;
         v.as_double = dp[ip];
         v.as_tags = v.as_tags | this_face_tag;
@@ -938,7 +939,7 @@ void ComputeSurfaceBoxes(GrGeomSolid *geom_solid)
       {
         int ip = SubvectorEltIndex(d_sub, i, j, k);
         int this_face_tag = 1 << PV_f;
-        
+
         DoubleTags v;
         v.as_double = dp[ip];
         v.as_tags = v.as_tags | this_face_tag;
diff --git a/pfsimulator/parflow_lib/clustering.h b/pfsimulator/parflow_lib/clustering.h
index e53e7e0ca..5f9a5daa3 100644
--- a/pfsimulator/parflow_lib/clustering.h
+++ b/pfsimulator/parflow_lib/clustering.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2019, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef _CLUSTERING_HEADER
 #define _CLUSTERING_HEADER
diff --git a/pfsimulator/parflow_lib/comm_pkg.c b/pfsimulator/parflow_lib/comm_pkg.c
index b11cd21f1..e636c5068 100644
--- a/pfsimulator/parflow_lib/comm_pkg.c
+++ b/pfsimulator/parflow_lib/comm_pkg.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
diff --git a/pfsimulator/parflow_lib/communication.c b/pfsimulator/parflow_lib/communication.c
index 1aaf91e9a..b5baacf55 100644
--- a/pfsimulator/parflow_lib/communication.c
+++ b/pfsimulator/parflow_lib/communication.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *****************************************************************************/
@@ -398,7 +398,7 @@ void FreeCommPkg(
 CommHandle  *InitCommunication(
                                CommPkg *comm_pkg)
 {
-  PUSH_NVTX("amps_IExchangePackage",6)
+  PUSH_NVTX("amps_IExchangePackage", 6)
   CommHandle* handle = (CommHandle*)amps_IExchangePackage(comm_pkg->package);
   POP_NVTX
 
@@ -413,8 +413,8 @@ CommHandle  *InitCommunication(
 void         FinalizeCommunication(
                                    CommHandle *handle)
 {
-  PUSH_NVTX("amps_Wait",1)
-  (void)amps_Wait((amps_Handle)handle);
+  PUSH_NVTX("amps_Wait", 1)
+    (void) amps_Wait((amps_Handle)handle);
   POP_NVTX
 }
 
diff --git a/pfsimulator/parflow_lib/communication.h b/pfsimulator/parflow_lib/communication.h
index f137521b7..4ec9bd5fe 100644
--- a/pfsimulator/parflow_lib/communication.h
+++ b/pfsimulator/parflow_lib/communication.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Header info for data structures for communication of information between
 * processes.
diff --git a/pfsimulator/parflow_lib/computation.c b/pfsimulator/parflow_lib/computation.c
index 54ed60cfb..ed669ecf6 100644
--- a/pfsimulator/parflow_lib/computation.c
+++ b/pfsimulator/parflow_lib/computation.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
diff --git a/pfsimulator/parflow_lib/computation.h b/pfsimulator/parflow_lib/computation.h
index 49a220ac1..bb030e1d2 100644
--- a/pfsimulator/parflow_lib/computation.h
+++ b/pfsimulator/parflow_lib/computation.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 * Header info for compute package data structures.
diff --git a/pfsimulator/parflow_lib/compute_maximums.c b/pfsimulator/parflow_lib/compute_maximums.c
index 740572f8c..a2362a336 100644
--- a/pfsimulator/parflow_lib/compute_maximums.c
+++ b/pfsimulator/parflow_lib/compute_maximums.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /***************************************************************************
 *
diff --git a/pfsimulator/parflow_lib/compute_patch_top.c b/pfsimulator/parflow_lib/compute_patch_top.c
index 124238784..af1972904 100644
--- a/pfsimulator/parflow_lib/compute_patch_top.c
+++ b/pfsimulator/parflow_lib/compute_patch_top.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*
  * Setup array for storing the patch indices for the top of the domain.
@@ -41,10 +41,9 @@
 #include "parflow.h"
 
 void ComputePatchTop(Problem *    problem,      /* General problem information */
-		     ProblemData *problem_data  /* Contains geometry information for the problem */
-		     )
+                     ProblemData *problem_data  /* Contains geometry information for the problem */
+                     )
 {
-
   Vector        *index_top = ProblemDataIndexOfDomainTop(problem_data);
   Vector        *patch_top = ProblemDataPatchIndexOfDomainTop(problem_data);
   Vector        *perm_x = ProblemDataPermeabilityX(problem_data);
@@ -61,7 +60,8 @@ void ComputePatchTop(Problem *    problem,      /* General problem information *
   double *patch_top_data;
   double *index_top_data;
 
-  int ipatch, ival=0;
+  int ipatch, ival = 0;
+
   PF_UNUSED(ival);
 
   VectorUpdateCommHandle   *handle;
@@ -87,61 +87,61 @@ void ComputePatchTop(Problem *    problem,      /* General problem information *
     ForSubgridI(is, grid3d_subgrids)
     {
       Subgrid       *grid2d_subgrid = SubgridArraySubgrid(grid2d_subgrids, is);
-      
+
       Subvector     *patch_top_subvector = VectorSubvector(patch_top, is);
       Subvector     *index_top_subvector = VectorSubvector(index_top, is);
-      
+
       int grid2d_iz = SubgridIZ(grid2d_subgrid);
-      
+
       patch_top_data = SubvectorData(patch_top_subvector);
       index_top_data = SubvectorData(index_top_subvector);
-      
+
       int i, j, k;
-      
+
       ForBCStructNumPatches(ipatch, bc_struct)
       {
-	ForPatchCellsPerFace(BC_ALL,
-			     BeforeAllCells(DoNothing),
-			     LoopVars(i, j, k, ival, bc_struct, ipatch, is),
-			     Locals(int current_patch_index, index2d;),
-			     CellSetup(
-                             {
-			       current_patch_index = -1;
-			       index2d = SubvectorEltIndex(patch_top_subvector, i, j, grid2d_iz);
-			     }),
-			     FACE(LeftFace, DoNothing),
-			     FACE(RightFace, DoNothing),
-			     FACE(DownFace, DoNothing),
-			     FACE(UpFace, DoNothing),
-			     FACE(BackFace, DoNothing),
-			     FACE(FrontFace,
-			     {
-			       if( index_top_data[index2d] > 0 )
-			       {
-				 current_patch_index = ipatch;
-			       }
-			     }),
-			     CellFinalize(
-			     {
-			       /* If we detected an UpperZFace that
-				  was on the top of the domain then
-				  set the patch index to the patch.
-				*/
-
-			       if (current_patch_index > -1)
-			       {
-				 //printf("Setting patch index %d, (%d,%d,%d)\n", ipatch, i, j, k);
-				 patch_top_data[index2d] = current_patch_index;
-			       }
-			     }),
-			     AfterAllCells(DoNothing)
-			     ); /* End BC_ALL */
+        ForPatchCellsPerFace(BC_ALL,
+                             BeforeAllCells(DoNothing),
+                             LoopVars(i, j, k, ival, bc_struct, ipatch, is),
+                             Locals(int current_patch_index, index2d; ),
+                             CellSetup(
+        {
+          current_patch_index = -1;
+          index2d = SubvectorEltIndex(patch_top_subvector, i, j, grid2d_iz);
+        }),
+                             FACE(LeftFace, DoNothing),
+                             FACE(RightFace, DoNothing),
+                             FACE(DownFace, DoNothing),
+                             FACE(UpFace, DoNothing),
+                             FACE(BackFace, DoNothing),
+                             FACE(FrontFace,
+        {
+          if (index_top_data[index2d] > 0)
+          {
+            current_patch_index = ipatch;
+          }
+        }),
+                             CellFinalize(
+        {
+          /* If we detected an UpperZFace that
+           * was on the top of the domain then
+           * set the patch index to the patch.
+           */
+
+          if (current_patch_index > -1)
+          {
+            //printf("Setting patch index %d, (%d,%d,%d)\n", ipatch, i, j, k);
+            patch_top_data[index2d] = current_patch_index;
+          }
+        }),
+                             AfterAllCells(DoNothing)
+                             ); /* End BC_ALL */
       } /* End ipatch loop */
     } /* End subgrid loop */
   } /* End num_patches > 0 */
 
   FreeBCStruct(bc_struct);
-    
+
   /* Pass top values to neighbors.  */
   handle = InitVectorUpdate(patch_top, VectorUpdateAll);
   FinalizeVectorUpdate(handle);
diff --git a/pfsimulator/parflow_lib/compute_top.c b/pfsimulator/parflow_lib/compute_top.c
index 6fbcdfa9c..567009725 100644
--- a/pfsimulator/parflow_lib/compute_top.c
+++ b/pfsimulator/parflow_lib/compute_top.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*
  * Setup array for storing the top of the domain.
diff --git a/pfsimulator/parflow_lib/compute_total_concentration.c b/pfsimulator/parflow_lib/compute_total_concentration.c
index f88d3e63d..0c6d62bb1 100644
--- a/pfsimulator/parflow_lib/compute_total_concentration.c
+++ b/pfsimulator/parflow_lib/compute_total_concentration.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *****************************************************************************/
diff --git a/pfsimulator/parflow_lib/constantRF.c b/pfsimulator/parflow_lib/constantRF.c
index 69333b9b6..6426cb501 100644
--- a/pfsimulator/parflow_lib/constantRF.c
+++ b/pfsimulator/parflow_lib/constantRF.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
diff --git a/pfsimulator/parflow_lib/constant_porosity.c b/pfsimulator/parflow_lib/constant_porosity.c
index 1dc2fa22a..56716f9e4 100644
--- a/pfsimulator/parflow_lib/constant_porosity.c
+++ b/pfsimulator/parflow_lib/constant_porosity.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
diff --git a/pfsimulator/parflow_lib/copy.c b/pfsimulator/parflow_lib/copy.c
index 6babec65d..cfbb85881 100644
--- a/pfsimulator/parflow_lib/copy.c
+++ b/pfsimulator/parflow_lib/copy.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
diff --git a/pfsimulator/parflow_lib/create_grid.c b/pfsimulator/parflow_lib/create_grid.c
index 51d3e9066..cae4d9754 100644
--- a/pfsimulator/parflow_lib/create_grid.c
+++ b/pfsimulator/parflow_lib/create_grid.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
diff --git a/pfsimulator/parflow_lib/diag_scale.c b/pfsimulator/parflow_lib/diag_scale.c
index 4530346bb..3cf3ea2c6 100644
--- a/pfsimulator/parflow_lib/diag_scale.c
+++ b/pfsimulator/parflow_lib/diag_scale.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
diff --git a/pfsimulator/parflow_lib/discretize_pressure.c b/pfsimulator/parflow_lib/discretize_pressure.c
index e6c0a00da..5935f5250 100644
--- a/pfsimulator/parflow_lib/discretize_pressure.c
+++ b/pfsimulator/parflow_lib/discretize_pressure.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
@@ -81,7 +81,7 @@ int seven_pt_shape[7][3] = { { 0, 0, 0 },
 #define Mean(a, b)  CellFaceConductivity(a, b)
 
 #define Coeff(Ta, Tb, Pa, Pb) \
-  (((Ta) + (Tb)) ? ((Ta) * (Pb) + (Tb) * (Pa)) / ((Ta) + (Tb)) : 0)
+        (((Ta) + (Tb)) ? ((Ta) * (Pb) + (Tb) * (Pa)) / ((Ta) + (Tb)) : 0)
 
 
 /*--------------------------------------------------------------------------
@@ -144,18 +144,18 @@ void          DiscretizePressure(
   Subvector     **tc_sub;
   Subvector      *tv_sub;
 
-  double dx, dy, dz, d=0.0;
+  double dx, dy, dz, d = 0.0;
 
   double         *cp, *wp, *ep, *sp, *np, *lp, *up, *op = NULL;
   double         *fp;
   double         *ttx_p, *tty_p, *ttz_p;
   double         *tmx_p, *tmy_p, *tmz_p;
-  double         *tm_p, *tt_p=NULL;
+  double         *tm_p, *tt_p = NULL;
   double        **tmx_pvec, **tmy_pvec, **tmz_pvec;
   double         *tc_p, **tc_pvec;
   double         *tv_p;
 
-  double scale, ffx, ffy, ffz, ff=0.0, vf;
+  double scale, ffx, ffy, ffz, ff = 0.0, vf;
 
   double e_temp, n_temp, u_temp, f_temp;
   double o_temp;
@@ -596,15 +596,15 @@ void          DiscretizePressure(
         d = dx;
         switch (fdir[0])
         {
-          case -1:
-            sv = -1;
-            op = wp;
-            break;
-
-          case  1:
-            sv = 1;
-            op = ep;
-            break;
+            case -1:
+              sv = -1;
+              op = wp;
+              break;
+
+            case  1:
+              sv = 1;
+              op = ep;
+              break;
         }
       }
       else if (fdir[1])
@@ -613,15 +613,15 @@ void          DiscretizePressure(
         d = dy;
         switch (fdir[1])
         {
-          case -1:
-            sv = -sy_v;
-            op = sp;
-            break;
-
-          case  1:
-            sv = sy_v;
-            op = np;
-            break;
+            case -1:
+              sv = -sy_v;
+              op = sp;
+              break;
+
+            case  1:
+              sv = sy_v;
+              op = np;
+              break;
         }
       }
       else if (fdir[2])
@@ -630,15 +630,15 @@ void          DiscretizePressure(
         d = dz;
         switch (fdir[2])
         {
-          case -1:
-            sv = -sz_v;
-            op = lp;
-            break;
-
-          case  1:
-            sv = sz_v;
-            op = up;
-            break;
+            case -1:
+              sv = -sz_v;
+              op = lp;
+              break;
+
+            case  1:
+              sv = sz_v;
+              op = up;
+              break;
         }
       }
 
@@ -744,79 +744,79 @@ void          DiscretizePressure(
                            LoopVars(i, j, k, ival, bc_struct, ipatch, is),
                            Locals(int iv, im, phase;
                                   double ff, d, o_temp, f_temp;
-                                  double *tm_p;),
+                                  double *tm_p; ),
                            CellSetup({
-                               iv = SubvectorEltIndex(f_sub, i, j, k);
-                               im = SubmatrixEltIndex(A_sub, i, j, k);
-                             }),
+        iv = SubvectorEltIndex(f_sub, i, j, k);
+        im = SubmatrixEltIndex(A_sub, i, j, k);
+      }),
                            FACE(LeftFace, {
-                               ff = ffx;
-                               d = dx;
-                               tt_p = ttx_p;
-                             }),
+        ff = ffx;
+        d = dx;
+        tt_p = ttx_p;
+      }),
                            FACE(RightFace, {
-                               ff = ffx;
-                               d = dx;
-                               tt_p = ttx_p;
-                             }),
+        ff = ffx;
+        d = dx;
+        tt_p = ttx_p;
+      }),
                            FACE(DownFace, {
-                               ff = ffy;
-                               d = dy;
-                               tt_p = tty_p;
-                             }),
+        ff = ffy;
+        d = dy;
+        tt_p = tty_p;
+      }),
                            FACE(UpFace, {
-                               ff = ffy;
-                               d = dy;
-                               tt_p = tty_p;
-                             }),
+        ff = ffy;
+        d = dy;
+        tt_p = tty_p;
+      }),
                            FACE(BackFace, {
-                               ff = ffz;
-                               d = dz;
-                               tt_p = ttz_p;
-
-                               for (phase = 0; phase < num_phases; phase++)
-                               {
-                                 tm_p = tmz_pvec[phase];
-                                 f_temp = ff * tm_p[iv] *
-                                          (phase_density[phase] * gravity);
-                                 fp[iv] += (-f_temp);
-                               }
-                             }),
+        ff = ffz;
+        d = dz;
+        tt_p = ttz_p;
+
+        for (phase = 0; phase < num_phases; phase++)
+        {
+          tm_p = tmz_pvec[phase];
+          f_temp = ff * tm_p[iv] *
+                   (phase_density[phase] * gravity);
+          fp[iv] += (-f_temp);
+        }
+      }),
                            FACE(FrontFace, {
-                               ff = ffz;
-                               d = dz;
-                               tt_p = ttz_p;
-
-                               for (phase = 0; phase < num_phases; phase++)
-                               {
-                                 tm_p = tmz_pvec[phase];
-                                 f_temp = ff * tm_p[iv] *
-                                          (phase_density[phase] * gravity);
-                                 fp[iv] += f_temp;
-                               }
-                             }),
+        ff = ffz;
+        d = dz;
+        tt_p = ttz_p;
+
+        for (phase = 0; phase < num_phases; phase++)
+        {
+          tm_p = tmz_pvec[phase];
+          f_temp = ff * tm_p[iv] *
+                   (phase_density[phase] * gravity);
+          fp[iv] += f_temp;
+        }
+      }),
                            CellFinalize({
-                               o_temp = -ff * 2.0 * tt_p[iv] / d;
-                               cp[im] -= o_temp;
-                               fp[iv] -= o_temp * bc_patch_values[ival];
-                             }),
+        o_temp = -ff * 2.0 * tt_p[iv] / d;
+        cp[im] -= o_temp;
+        fp[iv] -= o_temp * bc_patch_values[ival];
+      }),
                            AfterAllCells(DoNothing)
-        );
+                           );
 
       ForPatchCellsPerFace(FluxBC,
                            BeforeAllCells(DoNothing),
                            LoopVars(i, j, k, ival, bc_struct, ipatch, is),
-                           Locals(int iv, dir; double ff;),
+                           Locals(int iv, dir; double ff; ),
                            CellSetup({ iv = SubvectorEltIndex(f_sub, i, j, k); }),
-                           FACE(LeftFace,  { ff = ffx; dir = -1;}),
-                           FACE(RightFace, { ff = ffx; dir = 1;}),
-                           FACE(DownFace,  { ff = ffy; dir = -1;}),
-                           FACE(UpFace,    { ff = ffy; dir = 1;}),
-                           FACE(BackFace,  { ff = ffz; dir = -1;}),
-                           FACE(FrontFace, { ff = ffz; dir = 1;}),
+                           FACE(LeftFace, { ff = ffx; dir = -1; }),
+                           FACE(RightFace, { ff = ffx; dir = 1; }),
+                           FACE(DownFace, { ff = ffy; dir = -1; }),
+                           FACE(UpFace, { ff = ffy; dir = 1; }),
+                           FACE(BackFace, { ff = ffz; dir = -1; }),
+                           FACE(FrontFace, { ff = ffz; dir = 1; }),
                            CellFinalize({ fp[iv] -= ff * dir * bc_patch_values[ival]; }),
                            AfterAllCells(DoNothing)
-        );
+                           );
     }
   }
 
diff --git a/pfsimulator/parflow_lib/distribute_usergrid.c b/pfsimulator/parflow_lib/distribute_usergrid.c
index 6af4c3ecb..b6843fab1 100644
--- a/pfsimulator/parflow_lib/distribute_usergrid.c
+++ b/pfsimulator/parflow_lib/distribute_usergrid.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 * DistributeUserGrid
@@ -40,7 +40,7 @@
  * Macros for DistributeUserGrid
  *--------------------------------------------------------------------------*/
 
-#define pqr_to_xyz(pqr, mxyz, lxyz, xyz)   (pqr*mxyz + pfmin(pqr, lxyz) + xyz)
+#define pqr_to_xyz(pqr, mxyz, lxyz, xyz)   (pqr * mxyz + pfmin(pqr, lxyz) + xyz)
 
 #define pqr_to_nxyz(pqr, mxyz, lxyz)  (pqr < lxyz ? mxyz + 1 : mxyz)
 
diff --git a/pfsimulator/parflow_lib/evaptranssum.c b/pfsimulator/parflow_lib/evaptranssum.c
index f095a378a..2a9237a0e 100644
--- a/pfsimulator/parflow_lib/evaptranssum.c
+++ b/pfsimulator/parflow_lib/evaptranssum.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
diff --git a/pfsimulator/parflow_lib/fgetopt.c b/pfsimulator/parflow_lib/fgetopt.c
index ad6f5745d..01022f8fd 100644
--- a/pfsimulator/parflow_lib/fgetopt.c
+++ b/pfsimulator/parflow_lib/fgetopt.c
@@ -37,9 +37,9 @@
 #include <stdio.h>
 
 int opterr = 1,                 /* if error message should be printed */
-  optind = 1,                   /* index into parent argv vector */
-  optopt,                       /* character checked for validity */
-  optreset;                     /* reset getopt */
+    optind = 1,                 /* index into parent argv vector */
+    optopt,                     /* character checked for validity */
+    optreset;                   /* reset getopt */
 char    *optarg;                /* argument associated with option */
 
 #define BADCH   (int)'?'
@@ -119,15 +119,15 @@ I think you need to add extern char *optarg; in getopt.h as well to get that por
 @Marraz
 Marraz commented on Jun 17, 2016
 
-Actually, according to how http://pubs.opengroup.org/onlinepubs/9699919799/functions/getopt.html describe the functionality, the header should look like this, perhaps optreset not but I guess it doesn't hurt to have it there:
+Actually, according to how http: //pubs.opengroup.org/onlinepubs/9699919799/functions/getopt.html describe the functionality, the header should look like this, perhaps optreset not but I guess it doesn't hurt to have it there:
 ifndef GETOPT_H
 define GETOPT_H
 
-extern int opterr, /* if error message should be printed /
-                    * optind, / index into parent argv vector /
-                    * optopt, / character checked for validity /
-                    * optreset; / reset getopt /
-                    * extern char *optarg; / argument associated with option */
+extern int opterr,   /* if error message should be printed /
+                      * optind, / index into parent argv vector /
+                      * optopt, / character checked for validity /
+                      * optreset; / reset getopt /
+                      * extern char *optarg; / argument associated with option */
 
 int getopt(int nargc, char * const nargv[], const char *ostr);
 endif
diff --git a/pfsimulator/parflow_lib/file_versions.h b/pfsimulator/parflow_lib/file_versions.h
index 2dc80ecc8..f5b346eef 100644
--- a/pfsimulator/parflow_lib/file_versions.h
+++ b/pfsimulator/parflow_lib/file_versions.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef _FILE_VERSIONS_HEADER
 #define _FILE_VERSIONS_HEADER
diff --git a/pfsimulator/parflow_lib/general.c b/pfsimulator/parflow_lib/general.c
index fa8eaffcb..e8773593e 100644
--- a/pfsimulator/parflow_lib/general.c
+++ b/pfsimulator/parflow_lib/general.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
@@ -308,7 +308,7 @@ void printMemoryInfo(FILE *log_file)
 
   /* Get all the memory currently allocated to user by malloc, etc. */
   size_t used_mem = my_mallinfo.hblkhd + my_mallinfo.usmblks +
-                 my_mallinfo.uordblks;
+                    my_mallinfo.uordblks;
 
   /* Get memory not currently allocated to user but malloc controls */
   size_t free_mem = my_mallinfo.fsmblks + my_mallinfo.fordblks;
diff --git a/pfsimulator/parflow_lib/general.h b/pfsimulator/parflow_lib/general.h
index 044662197..2091bb3f8 100644
--- a/pfsimulator/parflow_lib/general.h
+++ b/pfsimulator/parflow_lib/general.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /** @file
  * @brief General macro definitions.
@@ -39,10 +39,10 @@
  * Error macros
  *--------------------------------------------------------------------------*/
 
-#define PARFLOW_ERROR(X)                \
-  do {                                  \
-    _amps_Abort(X, __FILE__, __LINE__); \
-  } while (0)
+#define PARFLOW_ERROR(X)                      \
+        do {                                  \
+          _amps_Abort(X, __FILE__, __LINE__); \
+        } while (0)
 
 /*--------------------------------------------------------------------------
  * Define memory allocation routines
@@ -52,7 +52,7 @@
  * @brief Allocates memory that is passed to amps library.
  *
  * When using an accelerator device, allocates unified memory with redefined macro.
- * 
+ *
  * @note Multiple definitions (see backend_mapping.h).
  *
  * @param type the C type name
@@ -65,7 +65,7 @@
  * @brief Allocates memory initialized to 0 that is passed to amps library.
  *
  * When using an accelerator device, allocates unified memory with redefined macro.
- * 
+ *
  * @note Multiple definitions (see backend_mapping.h).
  *
  * @param type the C type name
@@ -76,7 +76,7 @@
 
 /**
  * Deallocates memory for objects that were allocated by \ref talloc_amps_default or \ref ctalloc_amps_default.
- * 
+ *
  * @note Multiple definitions (see backend_mapping.h).
  *
  * @param ptr pointer to dataspace to free
@@ -91,11 +91,11 @@
  *--------------------------------------*/
 
 #define talloc_default(type, count) \
-  (type*)malloc_chk((unsigned int)((count) * sizeof(type)), __FILE__, __LINE__)
+        (type*)malloc_chk((unsigned int)((count) * sizeof(type)), __FILE__, __LINE__)
 
 #define ctalloc_default(type, count)                                           \
-  (type*)calloc_chk((unsigned int)(count), (unsigned int)sizeof(type), \
-                    __FILE__, __LINE__)
+        (type*)calloc_chk((unsigned int)(count), (unsigned int)sizeof(type),   \
+                          __FILE__, __LINE__)
 
 #else
 
@@ -107,7 +107,7 @@
  * @brief Allocates memory.
  *
  * When using an accelerator device, allocates unified memory with redefined macro.
- * 
+ *
  * @note Multiple definitions (see backend_mapping.h).
  *
  * @param type the C type name
@@ -115,13 +115,13 @@
  * @return pointer to the allocated dataspace
  */
 #define talloc_default(type, count) \
-  (((count) > 0) ? (type*)malloc(sizeof(type) * (unsigned int)(count)) : NULL)
+        (((count) > 0) ? (type*)malloc(sizeof(type) * (unsigned int)(count)) : NULL)
 
 /**
  * @brief Allocates memory initialized to 0.
  *
  * When using an accelerator device, allocates unified memory with redefined macro.
- * 
+ *
  * @note Multiple definitions (see backend_mapping.h).
  *
  * @param type the C type name
@@ -129,13 +129,13 @@
  * @return pointer to the allocated dataspace
  */
 #define ctalloc_default(type, count) \
-  (((count) > 0) ? (type*)calloc((unsigned int)(count), (unsigned int)sizeof(type)) : NULL)
+        (((count) > 0) ? (type*)calloc((unsigned int)(count), (unsigned int)sizeof(type)) : NULL)
 
 #endif
 
 /**
  * Deallocates memory for objects that were allocated by \ref talloc_default or \ref ctalloc_default.
- * 
+ *
  * @note Multiple definitions (see backend_mapping.h).
  *
  * @param ptr pointer to dataspace to free
@@ -145,7 +145,7 @@
 
 /**
  * Copies data from dest to src
- * 
+ *
  * @note Multiple definitions (see backend_mapping.h).
  *
  * @param dest destination address
@@ -178,7 +178,7 @@
 /**
  * Thread-safe addition assignment. This macro
  * can be called anywhere in any compute kernel.
- * 
+ *
  * @note Multiple definitions (see backend_mapping.h).
  *
  * @param a original value [IN], sum result [OUT]
@@ -189,29 +189,29 @@
 /**
  * Thread-safe reduction to find maximum value for reduction loops.
  * Each thread must call this macro as the last statement inside the reduction loop body.
- * 
+ *
  * @note Multiple definitions (see backend_mapping.h).
  *
  * @param a value 1 for comparison [IN], max value [OUT]
  * @param b value 2 for comparison [IN]
  */
-#define ReduceMax_default(a, b) if(a < b) { a = b; } else {};
+#define ReduceMax_default(a, b) if (a < b) { a = b; } else {};
 
 /**
  * Thread-safe reduction to find maximum value for reduction loops.
  * Each thread must call this macro as the last statement inside the reduction loop body.
- * 
+ *
  * @note Multiple definitions (see backend_mapping.h).
  *
  * @param a value 1 for comparison [IN], min value [OUT]
  * @param b value 2 for comparison [IN]
  */
-#define ReduceMin_default(a, b) if(a > b) { a = b; } else {};
+#define ReduceMin_default(a, b) if (a > b) { a = b; } else {};
 
 /**
  * Thread-safe addition assignment for reduction loops.
  * Each thread must call this macro as the last statement inside the reduction loop body.
- * 
+ *
  * @note Multiple definitions (see backend_mapping.h).
  *
  * @param a original value [IN], sum result [OUT]
@@ -261,25 +261,25 @@
  *--------------------------------------------------------------------------*/
 
 #ifndef __host__
-  /** Defines an object accessible from host. @note Does nothing if not supported by the compiler. */
+/** Defines an object accessible from host. @note Does nothing if not supported by the compiler. */
   #define __host__
 #endif
 #ifndef __device__
-  /** Defines an object accessible from device. @note Does nothing if not supported by the compiler. */
+/** Defines an object accessible from device. @note Does nothing if not supported by the compiler. */
   #define __device__
 #endif
 #ifndef __managed__
-  /** Defines a variable that is automatically migrated between host/device. @note Does nothing if not supported by the compiler. */
+/** Defines a variable that is automatically migrated between host/device. @note Does nothing if not supported by the compiler. */
   #define __managed__
 #endif
 #ifndef __restrict__
-  /** Defines a restricted pointer. @note Does nothing if not supported by the compiler. */
+/** Defines a restricted pointer. @note Does nothing if not supported by the compiler. */
   #define __restrict__
 #endif
 
 /**
  * Used to prefetch data from host to device for better performance.
- * 
+ *
  * @note Multiple definitions (see backend_mapping.h).
  *
  * @param ptr pointer to data [IN]
@@ -291,7 +291,7 @@
 
 /**
  * Used to prefetch data from host to device for better performance.
- * 
+ *
  * @note Multiple definitions (see backend_mapping.h).
  *
  * @param ptr pointer to data [IN]
@@ -304,7 +304,7 @@
 /**
  * Explicit sync between host and device default stream if accelerator present.
  * Can be called anywhere.
- * 
+ *
  * @note Multiple definitions (see backend_mapping.h).
  */
 #define PARALLEL_SYNC_default
@@ -312,13 +312,13 @@
 /**
  * Skip sync after BoxLoop if accelerator present.
  * Must be the called as the last action inside the loop body.
- * 
+ *
  * @note Multiple definitions (see backend_mapping.h).
  */
 #define SKIP_PARALLEL_SYNC_default
 
 /** Record an NVTX range for NSYS if accelerator present. @note Multiple definitions (see backend_mapping.h). */
-#define PUSH_NVTX_default(name,cid)
+#define PUSH_NVTX_default(name, cid)
 
 /** Stop recording an NVTX range for NSYS if accelerator present. @note Multiple definitions (see backend_mapping.h). */
 #define POP_NVTX_default
diff --git a/pfsimulator/parflow_lib/geom_t_solid.c b/pfsimulator/parflow_lib/geom_t_solid.c
index 5886e2e28..862f6eb30 100644
--- a/pfsimulator/parflow_lib/geom_t_solid.c
+++ b/pfsimulator/parflow_lib/geom_t_solid.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Member functions for the Geometry class.
diff --git a/pfsimulator/parflow_lib/geometry.c b/pfsimulator/parflow_lib/geometry.c
index 252553b70..15c2c74e2 100644
--- a/pfsimulator/parflow_lib/geometry.c
+++ b/pfsimulator/parflow_lib/geometry.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Member functions for the Geometry class.
diff --git a/pfsimulator/parflow_lib/geometry.h b/pfsimulator/parflow_lib/geometry.h
index 361ed165d..46c11f406 100644
--- a/pfsimulator/parflow_lib/geometry.h
+++ b/pfsimulator/parflow_lib/geometry.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Geometry class structures and accessors
 *
diff --git a/pfsimulator/parflow_lib/geostats.h b/pfsimulator/parflow_lib/geostats.h
index 006fd0025..1e8127215 100644
--- a/pfsimulator/parflow_lib/geostats.h
+++ b/pfsimulator/parflow_lib/geostats.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Geostatistics class structures
 *
diff --git a/pfsimulator/parflow_lib/globals.c b/pfsimulator/parflow_lib/globals.c
index 6972788c4..402f92011 100644
--- a/pfsimulator/parflow_lib/globals.c
+++ b/pfsimulator/parflow_lib/globals.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /** @file
  * @brief Routines for manipulating global structures.
diff --git a/pfsimulator/parflow_lib/globals.h b/pfsimulator/parflow_lib/globals.h
index 26d8e7c2a..902edb863 100644
--- a/pfsimulator/parflow_lib/globals.h
+++ b/pfsimulator/parflow_lib/globals.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *****************************************************************************/
diff --git a/pfsimulator/parflow_lib/grgeom_list.c b/pfsimulator/parflow_lib/grgeom_list.c
index 8d4cdfc9b..00b51cb40 100644
--- a/pfsimulator/parflow_lib/grgeom_list.c
+++ b/pfsimulator/parflow_lib/grgeom_list.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
diff --git a/pfsimulator/parflow_lib/grgeom_list.h b/pfsimulator/parflow_lib/grgeom_list.h
index 341d8089e..faaa84b10 100644
--- a/pfsimulator/parflow_lib/grgeom_list.h
+++ b/pfsimulator/parflow_lib/grgeom_list.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef _GRGEOM_LIST_HEADER
 #define _GRGEOM_LIST_HEADER
@@ -55,21 +55,21 @@ typedef struct list_member {
  * Macros : ListMember
  *--------------------------------------------------------------------------*/
 #define ListValueInClosedInterval(current_member, lower_point, upper_point) \
-  (((current_member) != NULL) ? (((ListMemberValue((current_member)) >= (lower_point)) && (ListMemberValue((current_member)) <= (upper_point))) ? TRUE : FALSE) : FALSE)
+        (((current_member) != NULL) ? (((ListMemberValue((current_member)) >= (lower_point)) && (ListMemberValue((current_member)) <= (upper_point))) ? TRUE : FALSE) : FALSE)
 
 #define ListValueLEPoint(current_member, point) \
-  (((current_member) != NULL) ? ((ListMemberValue((current_member)) <= (point)) ? TRUE : FALSE) : FALSE)
+        (((current_member) != NULL) ? ((ListMemberValue((current_member)) <= (point)) ? TRUE : FALSE) : FALSE)
 
 #define ListValueLTPoint(current_member, point) \
-  (((current_member) != NULL) ? ((ListMemberValue((current_member)) < (point)) ? TRUE : FALSE) : FALSE)
+        (((current_member) != NULL) ? ((ListMemberValue((current_member)) < (point)) ? TRUE : FALSE) : FALSE)
 
 #define ListValueEQPoint(current_member, point) \
-  (((current_member) != NULL) ? ((ListMemberValue((current_member)) == (point)) ? TRUE : FALSE) : FALSE)
+        (((current_member) != NULL) ? ((ListMemberValue((current_member)) == (point)) ? TRUE : FALSE) : FALSE)
 
 #define ListValueGTPoint(current_member, point) \
-  (((current_member) != NULL) ? ((ListMemberValue((current_member)) > (point)) ? TRUE : FALSE) : FALSE)
+        (((current_member) != NULL) ? ((ListMemberValue((current_member)) > (point)) ? TRUE : FALSE) : FALSE)
 
 #define ListValueGEPoint(current_member, point) \
-  (((current_member) != NULL) ? ((ListMemberValue((current_member)) >= (point)) ? TRUE : FALSE) : FALSE)
+        (((current_member) != NULL) ? ((ListMemberValue((current_member)) >= (point)) ? TRUE : FALSE) : FALSE)
 
 #endif
diff --git a/pfsimulator/parflow_lib/grgeom_octree.c b/pfsimulator/parflow_lib/grgeom_octree.c
index ddd651f3e..77d280f94 100644
--- a/pfsimulator/parflow_lib/grgeom_octree.c
+++ b/pfsimulator/parflow_lib/grgeom_octree.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -1659,7 +1659,7 @@ void    GrGeomOctreeFromInd(
 
         const int nx_f = SubvectorNX(subvector);
         const int ny_f = SubvectorNY(subvector);
-	const int nz_f = SubvectorNZ(subvector);
+        const int nz_f = SubvectorNZ(subvector);
 
         data = SubvectorElt(subvector, ix_all, iy_all, iz_all);
 
@@ -2334,21 +2334,21 @@ void          GrGeomPrintOctreeCells(
 
       switch (cell)
       {
-        case GrGeomOctreeNodeEmpty:
-          amps_Fprintf(file, "Empty\n");
-          break;
+          case GrGeomOctreeNodeEmpty:
+            amps_Fprintf(file, "Empty\n");
+            break;
 
-        case GrGeomOctreeNodeOutside:
-          amps_Fprintf(file, "Outside\n");
-          break;
+          case GrGeomOctreeNodeOutside:
+            amps_Fprintf(file, "Outside\n");
+            break;
 
-        case GrGeomOctreeNodeInside:
-          amps_Fprintf(file, "Inside\n");
-          break;
+          case GrGeomOctreeNodeInside:
+            amps_Fprintf(file, "Inside\n");
+            break;
 
-        case GrGeomOctreeNodeFull:
-          amps_Fprintf(file, "Full\n");
-          break;
+          case GrGeomOctreeNodeFull:
+            amps_Fprintf(file, "Full\n");
+            break;
       }
     },
     {
@@ -2358,21 +2358,21 @@ void          GrGeomPrintOctreeCells(
 
       switch (cell)
       {
-        case GrGeomOctreeNodeEmpty:
-          amps_Fprintf(file, "Empty\n");
-          break;
+          case GrGeomOctreeNodeEmpty:
+            amps_Fprintf(file, "Empty\n");
+            break;
 
-        case GrGeomOctreeNodeOutside:
-          amps_Fprintf(file, "Outside\n");
-          break;
+          case GrGeomOctreeNodeOutside:
+            amps_Fprintf(file, "Outside\n");
+            break;
 
-        case GrGeomOctreeNodeInside:
-          amps_Fprintf(file, "Inside\n");
-          break;
+          case GrGeomOctreeNodeInside:
+            amps_Fprintf(file, "Inside\n");
+            break;
 
-        case GrGeomOctreeNodeFull:
-          amps_Fprintf(file, "Full\n");
-          break;
+          case GrGeomOctreeNodeFull:
+            amps_Fprintf(file, "Full\n");
+            break;
       }
     });
   }
diff --git a/pfsimulator/parflow_lib/grgeom_octree.h b/pfsimulator/parflow_lib/grgeom_octree.h
index 1c36ffe64..e2adde9f4 100644
--- a/pfsimulator/parflow_lib/grgeom_octree.h
+++ b/pfsimulator/parflow_lib/grgeom_octree.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef _GRGEOM_OCTREE_HEADER
 #define _GRGEOM_OCTREE_HEADER
@@ -203,9 +203,9 @@ typedef struct grgeom_octree {
  * @param octree node to set
  * @param flag_value flag to set on node
  */
-#define GrGeomOctreeSetLeafFlag(octree, flag_value) \
-  GrGeomOctreeFlag(octree) =                        \
-    (GrGeomOctreeFlag(octree) & GrGeomOctreeNodeLeaf) | (flag_value)
+#define GrGeomOctreeSetLeafFlag(octree, flag_value)       \
+        GrGeomOctreeFlag(octree) =                        \
+          (GrGeomOctreeFlag(octree) &GrGeomOctreeNodeLeaf) | (flag_value)
 
 /**
  * @brief Set branch flag
@@ -215,9 +215,9 @@ typedef struct grgeom_octree {
  * @param octree node to set
  * @param flag_value flag to set on node
  */
-#define GrGeomOctreeSetBranchFlag(octree, flag_value) \
-  GrGeomOctreeFlag(octree) =                          \
-    (GrGeomOctreeFlag(octree) | (flag_value))
+#define GrGeomOctreeSetBranchFlag(octree, flag_value)       \
+        GrGeomOctreeFlag(octree) =                          \
+          (GrGeomOctreeFlag(octree) | (flag_value))
 
 /**
  * @brief Clear branch flag
@@ -228,7 +228,7 @@ typedef struct grgeom_octree {
  * @param flag_value flag to set clear node
  */
 #define GrGeomOctreeClearBranchFlag(octree, flag_value) \
-  GrGeomOctreeFlag(octree) &= ~(flag_value)
+        GrGeomOctreeFlag(octree) &= ~(flag_value)
 
 /**
  * @brief Set branch node flag value
@@ -240,7 +240,7 @@ typedef struct grgeom_octree {
  * @param flag_value flag to set on node
  */
 #define GrGeomOctreeSetBranchNode(octree, flag_value) \
-  GrGeomOctreeFlag(octree) |= (flag_value)
+        GrGeomOctreeFlag(octree) |= (flag_value)
 
 /**
  * @brief Clear the branch flag value
@@ -249,7 +249,7 @@ typedef struct grgeom_octree {
  * @param flag_value flag to clear
  */
 #define GrGeomOctreeClearBranchNode(octree, flag_value) \
-   GrGeomOctreeFlag(octree) &= ~(flag_value)
+        GrGeomOctreeFlag(octree) &= ~(flag_value)
 
 /**
  * @brief Set leaf indicator
@@ -259,7 +259,7 @@ typedef struct grgeom_octree {
  * @param octree node to make a leaf
  */
 #define GrGeomOctreeSetNodeLeaf(octree) \
-  (GrGeomOctreeFlag(octree) |= GrGeomOctreeNodeLeaf)
+        (GrGeomOctreeFlag(octree) |= GrGeomOctreeNodeLeaf)
 
 /**
  * @brief Clear leaf indicator and set to branch flag.
@@ -269,7 +269,7 @@ typedef struct grgeom_octree {
  * @param octree node to make a branch.
  */
 #define GrGeomOctreeClearNodeLeaf(octree) \
-  (GrGeomOctreeFlag(octree) &= ~GrGeomOctreeNodeLeaf)
+        (GrGeomOctreeFlag(octree) &= ~GrGeomOctreeNodeLeaf)
 
 /**
  * @name GrGeomOctreeFlagAccessors
@@ -280,11 +280,11 @@ typedef struct grgeom_octree {
  * @param face_index face index
  */
 #define GrGeomOctreeSetFace(octree, face_index) \
-  (GrGeomOctreeFaces(octree) |= GrGeomOctreeFaceValue(face_index))
+        (GrGeomOctreeFaces(octree) |= GrGeomOctreeFaceValue(face_index))
 #define GrGeomOctreeFaceValue(face_index) \
-  (((unsigned char)0x01) << face_index)
+        (((unsigned char)0x01) << face_index)
 #define GrGeomOctreeClearFace(octree, face_index) \
-  (GrGeomOctreeFaces(octree) &= ~GrGeomOctreeFaceValue(face_index))
+        (GrGeomOctreeFaces(octree) &= ~GrGeomOctreeFaceValue(face_index))
 /** @} */
 
 /**
@@ -298,7 +298,7 @@ typedef struct grgeom_octree {
  * @return face index
  */
 #define GrGeomOctreeFaceIndex(i, j, k) \
-  (int)(((i * (2 * i + 1)) + (j * (6 * j + 1)) + (k * (10 * k + 1))) / 2)
+        (int)(((i * (2 * i + 1)) + (j * (6 * j + 1)) + (k * (10 * k + 1))) / 2)
 
 /**
  * @name GrGeomOctreeStateTests
@@ -311,35 +311,35 @@ typedef struct grgeom_octree {
  * @{
  */
 #define GrGeomOctreeFlagIs(octree, flag_value) \
-  ((GrGeomOctreeFlag(octree) & flag_value) == flag_value)
+        ((GrGeomOctreeFlag(octree) &flag_value) == flag_value)
 #define GrGeomOctreeNodeIsEmpty(octree) \
-  GrGeomOctreeFlagIs(octree, GrGeomOctreeNodeEmpty)
+        GrGeomOctreeFlagIs(octree, GrGeomOctreeNodeEmpty)
 #define GrGeomOctreeNodeIsOutside(octree) \
-  GrGeomOctreeFlagIs(octree, GrGeomOctreeNodeOutside)
+        GrGeomOctreeFlagIs(octree, GrGeomOctreeNodeOutside)
 #define GrGeomOctreeNodeIsInside(octree) \
-  GrGeomOctreeFlagIs(octree, GrGeomOctreeNodeInside)
+        GrGeomOctreeFlagIs(octree, GrGeomOctreeNodeInside)
 #define GrGeomOctreeNodeIsFull(octree) \
-  GrGeomOctreeFlagIs(octree, GrGeomOctreeNodeFull)
+        GrGeomOctreeFlagIs(octree, GrGeomOctreeNodeFull)
 #define GrGeomOctreeNodeIsLeaf(octree) \
-  GrGeomOctreeFlagIs(octree, GrGeomOctreeNodeLeaf)
+        GrGeomOctreeFlagIs(octree, GrGeomOctreeNodeLeaf)
 #define GrGeomOctreeHasChildren(octree) \
-  (GrGeomOctreeChildren(octree) != NULL)
+        (GrGeomOctreeChildren(octree) != NULL)
 #define GrGeomOctreeHasNoChildren(octree) \
-  (GrGeomOctreeChildren(octree) == NULL)
+        (GrGeomOctreeChildren(octree) == NULL)
 #define GrGeomOctreeHasChild(octree, i) \
-  (GrGeomOctreeChild(octree, i) != NULL)
+        (GrGeomOctreeChild(octree, i) != NULL)
 #define GrGeomOctreeHasNoChild(octree, i) \
-  (GrGeomOctreeChild(octree, i) == NULL)
+        (GrGeomOctreeChild(octree, i) == NULL)
 #define GrGeomOctreeHasFaces(octree) \
-  (GrGeomOctreeFaces(octree) != ((unsigned char)0x00))
+        (GrGeomOctreeFaces(octree) != ((unsigned char)0x00))
 #define GrGeomOctreeHasNoFaces(octree) \
-  (GrGeomOctreeFaces(octree) == ((unsigned char)0x00))
-#define GrGeomOctreeHasFace(octree, face_index)                       \
-  ((GrGeomOctreeFaces(octree) & GrGeomOctreeFaceValue(face_index)) != \
-   ((unsigned char)0x00))
-#define GrGeomOctreeHasNoFace(octree, face_index)                     \
-  ((GrGeomOctreeFaces(octree) & GrGeomOctreeFaceValue(face_index)) == \
-   ((unsigned char)0x00))
+        (GrGeomOctreeFaces(octree) == ((unsigned char)0x00))
+#define GrGeomOctreeHasFace(octree, face_index)                            \
+        ((GrGeomOctreeFaces(octree) &GrGeomOctreeFaceValue(face_index)) != \
+         ((unsigned char)0x00))
+#define GrGeomOctreeHasNoFace(octree, face_index)                          \
+        ((GrGeomOctreeFaces(octree) &GrGeomOctreeFaceValue(face_index)) == \
+         ((unsigned char)0x00))
 /** @} */
 
 /**
@@ -369,74 +369,74 @@ typedef struct grgeom_octree {
  * @param[in] level_body code to execute if level has been reached.
  * @param[in] leaf_body code to execute for leaf nodes
  */
-#define GrGeomOctreeLoop(i, j, k, l, node, octree, level, value_test, \
-                         level_body, leaf_body)                       \
-  {                                                                   \
-    int PV_level = level;                                             \
-    unsigned int PV_inc;                                              \
-    int           *PV_visiting;                                       \
-    int PV_visit_child;                                               \
-                                                                      \
-                                                                      \
-    node = octree;                                                    \
-                                                                      \
-    l = 0;                                                            \
-    PV_inc = 1 << PV_level;                                           \
-    PV_visiting = ctalloc(int, PV_level + 2);                         \
-    PV_visiting++;                                                    \
-    PV_visiting[0] = 0;                                               \
-                                                                      \
-    while (l >= 0)                                                    \
-    {                                                                 \
-      /* if at the level of interest */                               \
-      if (l == PV_level)                                              \
-      {                                                               \
-        if (value_test)                                               \
-          level_body;                                                 \
-                                                                      \
-        PV_visit_child = FALSE;                                       \
-      }                                                               \
-                                                                      \
-      /* if this is a leaf node */                                    \
-      else if (GrGeomOctreeNodeIsLeaf(node))                          \
-      {                                                               \
-        if (value_test)                                               \
-          leaf_body;                                                  \
-                                                                      \
-        PV_visit_child = FALSE;                                       \
-      }                                                               \
-                                                                      \
-      /* have I visited all of the children? */                       \
-      else if (PV_visiting[l] < GrGeomOctreeNumChildren)              \
-        PV_visit_child = TRUE;                                        \
-      else                                                            \
-        PV_visit_child = FALSE;                                       \
-                                                                      \
-      /* visit either a child or the parent node */                   \
-      if (PV_visit_child)                                             \
-      {                                                               \
-        node = GrGeomOctreeChild(node, PV_visiting[l]);               \
-        PV_inc = PV_inc >> 1;                                         \
-        i += (int)(PV_inc) * ((PV_visiting[l] & 1) ? 1 : 0);          \
-        j += (int)(PV_inc) * ((PV_visiting[l] & 2) ? 1 : 0);          \
-        k += (int)(PV_inc) * ((PV_visiting[l] & 4) ? 1 : 0);          \
-        l++;                                                          \
-        PV_visiting[l] = 0;                                           \
-      }                                                               \
-      else                                                            \
-      {                                                               \
-        l--;                                                          \
-        i -= (int)(PV_inc) * ((PV_visiting[l] & 1) ? 1 : 0);          \
-        j -= (int)(PV_inc) * ((PV_visiting[l] & 2) ? 1 : 0);          \
-        k -= (int)(PV_inc) * ((PV_visiting[l] & 4) ? 1 : 0);          \
-        PV_inc = PV_inc << 1;                                         \
-        node = GrGeomOctreeParent(node);                              \
-        PV_visiting[l]++;                                             \
-      }                                                               \
-    }                                                                 \
-                                                                      \
-    tfree(PV_visiting - 1);                                           \
-  }
+#define GrGeomOctreeLoop(i, j, k, l, node, octree, level, value_test,       \
+                         level_body, leaf_body)                             \
+        {                                                                   \
+          int PV_level = level;                                             \
+          unsigned int PV_inc;                                              \
+          int           *PV_visiting;                                       \
+          int PV_visit_child;                                               \
+                                                                            \
+                                                                            \
+          node = octree;                                                    \
+                                                                            \
+          l = 0;                                                            \
+          PV_inc = 1 << PV_level;                                           \
+          PV_visiting = ctalloc(int, PV_level + 2);                         \
+          PV_visiting++;                                                    \
+          PV_visiting[0] = 0;                                               \
+                                                                            \
+          while (l >= 0)                                                    \
+          {                                                                 \
+            /* if at the level of interest */                               \
+            if (l == PV_level)                                              \
+            {                                                               \
+              if (value_test)                                               \
+              level_body;                                                   \
+                                                                            \
+              PV_visit_child = FALSE;                                       \
+            }                                                               \
+                                                                            \
+            /* if this is a leaf node */                                    \
+            else if (GrGeomOctreeNodeIsLeaf(node))                          \
+            {                                                               \
+              if (value_test)                                               \
+              leaf_body;                                                    \
+                                                                            \
+              PV_visit_child = FALSE;                                       \
+            }                                                               \
+                                                                            \
+            /* have I visited all of the children? */                       \
+            else if (PV_visiting[l] < GrGeomOctreeNumChildren)              \
+            PV_visit_child = TRUE;                                          \
+            else                                                            \
+            PV_visit_child = FALSE;                                         \
+                                                                            \
+            /* visit either a child or the parent node */                   \
+            if (PV_visit_child)                                             \
+            {                                                               \
+              node = GrGeomOctreeChild(node, PV_visiting[l]);               \
+              PV_inc = PV_inc >> 1;                                         \
+              i += (int)(PV_inc) * ((PV_visiting[l] & 1) ? 1 : 0);          \
+              j += (int)(PV_inc) * ((PV_visiting[l] & 2) ? 1 : 0);          \
+              k += (int)(PV_inc) * ((PV_visiting[l] & 4) ? 1 : 0);          \
+              l++;                                                          \
+              PV_visiting[l] = 0;                                           \
+            }                                                               \
+            else                                                            \
+            {                                                               \
+              l--;                                                          \
+              i -= (int)(PV_inc) * ((PV_visiting[l] & 1) ? 1 : 0);          \
+              j -= (int)(PV_inc) * ((PV_visiting[l] & 2) ? 1 : 0);          \
+              k -= (int)(PV_inc) * ((PV_visiting[l] & 4) ? 1 : 0);          \
+              PV_inc = PV_inc << 1;                                         \
+              node = GrGeomOctreeParent(node);                              \
+              PV_visiting[l]++;                                             \
+            }                                                               \
+          }                                                                 \
+                                                                            \
+          tfree(PV_visiting - 1);                                           \
+        }
 
 /**
  * @brief Internal generic octree looping
@@ -470,73 +470,73 @@ typedef struct grgeom_octree {
  * @param[in] traversal_stop_test boolean expression evalutated to determine if a branch node should terminate traversal
  *  @param[in] body_test boolean expression evalutated to determine if code_body should be executed.
  */
-#define GrGeomOctreeLoopTemplate(i, j, k, l, node, octree, level, value_test,           \
-                                 level_body, code_body, traversal_stop_test, body_test) \
-  {                                                                                     \
-    int PV_level = level;                                                               \
-    unsigned int PV_inc;                                                                \
-    int           *PV_visiting;                                                         \
-    int PV_visit_child;                                                                 \
-                                                                                        \
-    node = octree;                                                                      \
-                                                                                        \
-    l = 0;                                                                              \
-    PV_inc = 1 << PV_level;                                                             \
-    PV_visiting = ctalloc(int, PV_level + 2);                                           \
-    PV_visiting++;                                                                      \
-    PV_visiting[0] = 0;                                                                 \
-                                                                                        \
-    while (l >= 0)                                                                      \
-    {                                                                                   \
-      /* if at the level of interest */                                                 \
-      if (l == PV_level)                                                                \
-      {                                                                                 \
-        if ((body_test) && (value_test))                                                \
-          level_body;                                                                   \
-                                                                                        \
-        PV_visit_child = FALSE;                                                         \
-      }                                                                                 \
-                                                                                        \
-      /* if this is a full node or a leaf node */                                       \
-      else if (GrGeomOctreeNodeIsLeaf(node) || (traversal_stop_test))                   \
-      {                                                                                 \
-        if ((body_test) && (value_test))                                                \
-          code_body;                                                                    \
-                                                                                        \
-        PV_visit_child = FALSE;                                                         \
-      }                                                                                 \
-                                                                                        \
-      /* have I visited all of the children? */                                         \
-      else if (PV_visiting[l] < GrGeomOctreeNumChildren)                                \
-        PV_visit_child = TRUE;                                                          \
-      else                                                                              \
-        PV_visit_child = FALSE;                                                         \
-                                                                                        \
-      /* visit either a child or the parent node */                                     \
-      if (PV_visit_child)                                                               \
-      {                                                                                 \
-        node = GrGeomOctreeChild(node, PV_visiting[l]);                                 \
-        PV_inc = PV_inc >> 1;                                                           \
-        i += (int)(PV_inc) * ((PV_visiting[l] & 1) ? 1 : 0);                            \
-        j += (int)(PV_inc) * ((PV_visiting[l] & 2) ? 1 : 0);                            \
-        k += (int)(PV_inc) * ((PV_visiting[l] & 4) ? 1 : 0);                            \
-        l++;                                                                            \
-        PV_visiting[l] = 0;                                                             \
-      }                                                                                 \
-      else                                                                              \
-      {                                                                                 \
-        l--;                                                                            \
-        i -= (int)(PV_inc) * ((PV_visiting[l] & 1) ? 1 : 0);                            \
-        j -= (int)(PV_inc) * ((PV_visiting[l] & 2) ? 1 : 0);                            \
-        k -= (int)(PV_inc) * ((PV_visiting[l] & 4) ? 1 : 0);                            \
-        PV_inc = PV_inc << 1;                                                           \
-        node = GrGeomOctreeParent(node);                                                \
-        PV_visiting[l]++;                                                               \
-      }                                                                                 \
-    }                                                                                   \
-                                                                                        \
-    tfree(PV_visiting - 1);                                                             \
-  }
+#define GrGeomOctreeLoopTemplate(i, j, k, l, node, octree, level, value_test,                 \
+                                 level_body, code_body, traversal_stop_test, body_test)       \
+        {                                                                                     \
+          int PV_level = level;                                                               \
+          unsigned int PV_inc;                                                                \
+          int           *PV_visiting;                                                         \
+          int PV_visit_child;                                                                 \
+                                                                                              \
+          node = octree;                                                                      \
+                                                                                              \
+          l = 0;                                                                              \
+          PV_inc = 1 << PV_level;                                                             \
+          PV_visiting = ctalloc(int, PV_level + 2);                                           \
+          PV_visiting++;                                                                      \
+          PV_visiting[0] = 0;                                                                 \
+                                                                                              \
+          while (l >= 0)                                                                      \
+          {                                                                                   \
+            /* if at the level of interest */                                                 \
+            if (l == PV_level)                                                                \
+            {                                                                                 \
+              if ((body_test) && (value_test))                                                \
+              level_body;                                                                     \
+                                                                                              \
+              PV_visit_child = FALSE;                                                         \
+            }                                                                                 \
+                                                                                              \
+            /* if this is a full node or a leaf node */                                       \
+            else if (GrGeomOctreeNodeIsLeaf(node) || (traversal_stop_test))                   \
+            {                                                                                 \
+              if ((body_test) && (value_test))                                                \
+              code_body;                                                                      \
+                                                                                              \
+              PV_visit_child = FALSE;                                                         \
+            }                                                                                 \
+                                                                                              \
+            /* have I visited all of the children? */                                         \
+            else if (PV_visiting[l] < GrGeomOctreeNumChildren)                                \
+            PV_visit_child = TRUE;                                                            \
+            else                                                                              \
+            PV_visit_child = FALSE;                                                           \
+                                                                                              \
+            /* visit either a child or the parent node */                                     \
+            if (PV_visit_child)                                                               \
+            {                                                                                 \
+              node = GrGeomOctreeChild(node, PV_visiting[l]);                                 \
+              PV_inc = PV_inc >> 1;                                                           \
+              i += (int)(PV_inc) * ((PV_visiting[l] & 1) ? 1 : 0);                            \
+              j += (int)(PV_inc) * ((PV_visiting[l] & 2) ? 1 : 0);                            \
+              k += (int)(PV_inc) * ((PV_visiting[l] & 4) ? 1 : 0);                            \
+              l++;                                                                            \
+              PV_visiting[l] = 0;                                                             \
+            }                                                                                 \
+            else                                                                              \
+            {                                                                                 \
+              l--;                                                                            \
+              i -= (int)(PV_inc) * ((PV_visiting[l] & 1) ? 1 : 0);                            \
+              j -= (int)(PV_inc) * ((PV_visiting[l] & 2) ? 1 : 0);                            \
+              k -= (int)(PV_inc) * ((PV_visiting[l] & 4) ? 1 : 0);                            \
+              PV_inc = PV_inc << 1;                                                           \
+              node = GrGeomOctreeParent(node);                                                \
+              PV_visiting[l]++;                                                               \
+            }                                                                                 \
+          }                                                                                   \
+                                                                                              \
+          tfree(PV_visiting - 1);                                                             \
+        }
 
 
 /**
@@ -570,14 +570,14 @@ typedef struct grgeom_octree {
  * @param[in] level_body code to execute if level has been reached.
  * @param[in] code_body code to execute for interior nodes
  */
-#define GrGeomOctreeInteriorLoop(i, j, k, l, node, octree, level, value_test,                   \
-                                 level_body, code_body)                                         \
-  {                                                                                             \
-    GrGeomOctreeLoopTemplate(i, j, k, l, node, octree, level, value_test,                       \
-                             level_body, code_body,                                             \
-                             GrGeomOctreeNodeIsFull(node),                                      \
-                             (GrGeomOctreeNodeIsInside(node) || GrGeomOctreeNodeIsFull(node))); \
-  }
+#define GrGeomOctreeInteriorLoop(i, j, k, l, node, octree, level, value_test,                         \
+                                 level_body, code_body)                                               \
+        {                                                                                             \
+          GrGeomOctreeLoopTemplate(i, j, k, l, node, octree, level, value_test,                       \
+                                   level_body, code_body,                                             \
+                                   GrGeomOctreeNodeIsFull(node),                                      \
+                                   (GrGeomOctreeNodeIsInside(node) || GrGeomOctreeNodeIsFull(node))); \
+        }
 
 /**
  * @brief Internal use generic octree looping for Exteriror indices only.
@@ -610,14 +610,14 @@ typedef struct grgeom_octree {
  * @param[in] level_body code to execute if level has been reached.
  * @param[in] code_body code to execute for exterior nodes
  */
-#define GrGeomOctreeExteriorLoop(i, j, k, l, node, octree, level, value_test,                     \
-                                 level_body, code_body)                                           \
-  {                                                                                               \
-    GrGeomOctreeLoopTemplate(i, j, k, l, node, octree, level, value_test,                         \
-                             level_body, code_body,                                               \
-                             GrGeomOctreeNodeIsEmpty(node),                                       \
-                             (GrGeomOctreeNodeIsOutside(node) || GrGeomOctreeNodeIsEmpty(node))); \
-  }
+#define GrGeomOctreeExteriorLoop(i, j, k, l, node, octree, level, value_test,                           \
+                                 level_body, code_body)                                                 \
+        {                                                                                               \
+          GrGeomOctreeLoopTemplate(i, j, k, l, node, octree, level, value_test,                         \
+                                   level_body, code_body,                                               \
+                                   GrGeomOctreeNodeIsEmpty(node),                                       \
+                                   (GrGeomOctreeNodeIsOutside(node) || GrGeomOctreeNodeIsEmpty(node))); \
+        }
 
 /**
  * @brief Internal use generic octree looping for Inside indices only.
@@ -650,79 +650,79 @@ typedef struct grgeom_octree {
  * @param[in] level_body code to execute if level has been reached.
  * @param[in] code_body code to execute for inside nodes
  */
-#define GrGeomOctreeInsideLoop(i, j, k, l, node, octree, level, value_test, \
-                               level_body, code_body)                       \
-  {                                                                         \
-    int PV_level = level;                                                   \
-    unsigned int PV_inc;                                                    \
-    int           *PV_visiting;                                             \
-    int PV_visit_child;                                                     \
-                                                                            \
-    node = octree;                                                          \
-                                                                            \
-    l = 0;                                                                  \
-    PV_inc = 1 << PV_level;                                                 \
-    PV_visiting = ctalloc(int, PV_level + 2);                               \
-    PV_visiting++;                                                          \
-    PV_visiting[0] = 0;                                                     \
-                                                                            \
-    while (l >= 0)                                                          \
-    {                                                                       \
-      /* if at the level of interest */                                     \
-      if (l == PV_level)                                                    \
-      {                                                                     \
-        if ((GrGeomOctreeNodeIsInside(node)) && (value_test))               \
-          level_body;                                                       \
-                                                                            \
-        PV_visit_child = FALSE;                                             \
-      }                                                                     \
-                                                                            \
-      /* if this is a full node or a leaf node */                           \
-      else if (GrGeomOctreeNodeIsLeaf(node))                                \
-      {                                                                     \
-        if (GrGeomOctreeNodeIsInside(node) && (value_test))                 \
-          code_body;                                                        \
-                                                                            \
-        PV_visit_child = FALSE;                                             \
-      }                                                                     \
-                                                                            \
-      /* Branch node and not inside, don't need to continue traversal */    \
-      else if (!GrGeomOctreeNodeIsInside(node))                             \
-      {                                                                     \
-        PV_visit_child = FALSE;                                             \
-      }                                                                     \
-                                                                            \
-      /* have I visited all of the children? */                             \
-      else if (PV_visiting[l] < GrGeomOctreeNumChildren)                    \
-        PV_visit_child = TRUE;                                              \
-      else                                                                  \
-        PV_visit_child = FALSE;                                             \
-                                                                            \
-      /* visit either a child or the parent node */                         \
-      if (PV_visit_child)                                                   \
-      {                                                                     \
-        node = GrGeomOctreeChild(node, PV_visiting[l]);                     \
-        PV_inc = PV_inc >> 1;                                               \
-        i += (int)(PV_inc) * ((PV_visiting[l] & 1) ? 1 : 0);                \
-        j += (int)(PV_inc) * ((PV_visiting[l] & 2) ? 1 : 0);                \
-        k += (int)(PV_inc) * ((PV_visiting[l] & 4) ? 1 : 0);                \
-        l++;                                                                \
-        PV_visiting[l] = 0;                                                 \
-      }                                                                     \
-      else                                                                  \
-      {                                                                     \
-        l--;                                                                \
-        i -= (int)(PV_inc) * ((PV_visiting[l] & 1) ? 1 : 0);                \
-        j -= (int)(PV_inc) * ((PV_visiting[l] & 2) ? 1 : 0);                \
-        k -= (int)(PV_inc) * ((PV_visiting[l] & 4) ? 1 : 0);                \
-        PV_inc = PV_inc << 1;                                               \
-        node = GrGeomOctreeParent(node);                                    \
-        PV_visiting[l]++;                                                   \
-      }                                                                     \
-    }                                                                       \
-                                                                            \
-    tfree(PV_visiting - 1);                                                 \
-  }
+#define GrGeomOctreeInsideLoop(i, j, k, l, node, octree, level, value_test,       \
+                               level_body, code_body)                             \
+        {                                                                         \
+          int PV_level = level;                                                   \
+          unsigned int PV_inc;                                                    \
+          int           *PV_visiting;                                             \
+          int PV_visit_child;                                                     \
+                                                                                  \
+          node = octree;                                                          \
+                                                                                  \
+          l = 0;                                                                  \
+          PV_inc = 1 << PV_level;                                                 \
+          PV_visiting = ctalloc(int, PV_level + 2);                               \
+          PV_visiting++;                                                          \
+          PV_visiting[0] = 0;                                                     \
+                                                                                  \
+          while (l >= 0)                                                          \
+          {                                                                       \
+            /* if at the level of interest */                                     \
+            if (l == PV_level)                                                    \
+            {                                                                     \
+              if ((GrGeomOctreeNodeIsInside(node)) && (value_test))               \
+              level_body;                                                         \
+                                                                                  \
+              PV_visit_child = FALSE;                                             \
+            }                                                                     \
+                                                                                  \
+            /* if this is a full node or a leaf node */                           \
+            else if (GrGeomOctreeNodeIsLeaf(node))                                \
+            {                                                                     \
+              if (GrGeomOctreeNodeIsInside(node) && (value_test))                 \
+              code_body;                                                          \
+                                                                                  \
+              PV_visit_child = FALSE;                                             \
+            }                                                                     \
+                                                                                  \
+            /* Branch node and not inside, don't need to continue traversal */    \
+            else if (!GrGeomOctreeNodeIsInside(node))                             \
+            {                                                                     \
+              PV_visit_child = FALSE;                                             \
+            }                                                                     \
+                                                                                  \
+            /* have I visited all of the children? */                             \
+            else if (PV_visiting[l] < GrGeomOctreeNumChildren)                    \
+            PV_visit_child = TRUE;                                                \
+            else                                                                  \
+            PV_visit_child = FALSE;                                               \
+                                                                                  \
+            /* visit either a child or the parent node */                         \
+            if (PV_visit_child)                                                   \
+            {                                                                     \
+              node = GrGeomOctreeChild(node, PV_visiting[l]);                     \
+              PV_inc = PV_inc >> 1;                                               \
+              i += (int)(PV_inc) * ((PV_visiting[l] & 1) ? 1 : 0);                \
+              j += (int)(PV_inc) * ((PV_visiting[l] & 2) ? 1 : 0);                \
+              k += (int)(PV_inc) * ((PV_visiting[l] & 4) ? 1 : 0);                \
+              l++;                                                                \
+              PV_visiting[l] = 0;                                                 \
+            }                                                                     \
+            else                                                                  \
+            {                                                                     \
+              l--;                                                                \
+              i -= (int)(PV_inc) * ((PV_visiting[l] & 1) ? 1 : 0);                \
+              j -= (int)(PV_inc) * ((PV_visiting[l] & 2) ? 1 : 0);                \
+              k -= (int)(PV_inc) * ((PV_visiting[l] & 4) ? 1 : 0);                \
+              PV_inc = PV_inc << 1;                                               \
+              node = GrGeomOctreeParent(node);                                    \
+              PV_visiting[l]++;                                                   \
+            }                                                                     \
+          }                                                                       \
+                                                                                  \
+          tfree(PV_visiting - 1);                                                 \
+        }
 
 /**
  * @brief Loop over the interior index space of an octree.
@@ -760,48 +760,48 @@ typedef struct grgeom_octree {
  * @param[in] value_test boolean if tests evaluated before body execution
  * @param[in] body code to execute
  */
-#define GrGeomOctreeInteriorNodeLoop(i, j, k, node, octree, level,                    \
-                                     ix, iy, iz, nx, ny, nz, value_test,              \
-                                     body)                                            \
-  {                                                                                   \
-    int PV_i, PV_j, PV_k, PV_l;                                                       \
-    int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                               \
-                                                                                      \
-                                                                                      \
-    PV_i = i;                                                                         \
-    PV_j = j;                                                                         \
-    PV_k = k;                                                                         \
-                                                                                      \
-    GrGeomOctreeInteriorLoop(PV_i, PV_j, PV_k, PV_l, node, octree, level, value_test, \
-    {                                                                                 \
-      if ((PV_i >= ix) && (PV_i < (ix + nx)) &&                                       \
-          (PV_j >= iy) && (PV_j < (iy + ny)) &&                                       \
-          (PV_k >= iz) && (PV_k < (iz + nz)))                                         \
-      {                                                                               \
-        i = PV_i;                                                                     \
-        j = PV_j;                                                                     \
-        k = PV_k;                                                                     \
-        body;                                                                         \
-      }                                                                               \
-    },                                                                                \
-    {                                                                                 \
-      /* find octree and region intersection */                                       \
-      PV_ixl = pfmax(ix, PV_i);                                                       \
-      PV_iyl = pfmax(iy, PV_j);                                                       \
-      PV_izl = pfmax(iz, PV_k);                                                       \
-      PV_ixu = pfmin((ix + nx), (PV_i + (int)PV_inc));                                \
-      PV_iyu = pfmin((iy + ny), (PV_j + (int)PV_inc));                                \
-      PV_izu = pfmin((iz + nz), (PV_k + (int)PV_inc));                                \
-                                                                                      \
-      /* loop over indexes and execute the body */                                    \
-      for (k = PV_izl; k < PV_izu; k++)                                               \
-        for (j = PV_iyl; j < PV_iyu; j++)                                             \
-          for (i = PV_ixl; i < PV_ixu; i++)                                           \
-          {                                                                           \
-            body;                                                                     \
-          }                                                                           \
-    })                                                                                \
-  }
+#define GrGeomOctreeInteriorNodeLoop(i, j, k, node, octree, level,                          \
+                                     ix, iy, iz, nx, ny, nz, value_test,                    \
+                                     body)                                                  \
+        {                                                                                   \
+          int PV_i, PV_j, PV_k, PV_l;                                                       \
+          int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                               \
+                                                                                            \
+                                                                                            \
+          PV_i = i;                                                                         \
+          PV_j = j;                                                                         \
+          PV_k = k;                                                                         \
+                                                                                            \
+          GrGeomOctreeInteriorLoop(PV_i, PV_j, PV_k, PV_l, node, octree, level, value_test, \
+    {                                                                                       \
+      if ((PV_i >= ix) && (PV_i < (ix + nx)) &&                                             \
+          (PV_j >= iy) && (PV_j < (iy + ny)) &&                                             \
+          (PV_k >= iz) && (PV_k < (iz + nz)))                                               \
+      {                                                                                     \
+        i = PV_i;                                                                           \
+        j = PV_j;                                                                           \
+        k = PV_k;                                                                           \
+        body;                                                                               \
+      }                                                                                     \
+    },                                                                                      \
+    {                                                                                       \
+      /* find octree and region intersection */                                             \
+      PV_ixl = pfmax(ix, PV_i);                                                             \
+      PV_iyl = pfmax(iy, PV_j);                                                             \
+      PV_izl = pfmax(iz, PV_k);                                                             \
+      PV_ixu = pfmin((ix + nx), (PV_i + (int)PV_inc));                                      \
+      PV_iyu = pfmin((iy + ny), (PV_j + (int)PV_inc));                                      \
+      PV_izu = pfmin((iz + nz), (PV_k + (int)PV_inc));                                      \
+                                                                                            \
+      /* loop over indexes and execute the body */                                          \
+      for (k = PV_izl; k < PV_izu; k++)                                                     \
+      for (j = PV_iyl; j < PV_iyu; j++)                                                     \
+      for (i = PV_ixl; i < PV_ixu; i++)                                                     \
+      {                                                                                     \
+        body;                                                                               \
+      }                                                                                     \
+    })                                                                                      \
+        }
 
 
 /**
@@ -840,48 +840,48 @@ typedef struct grgeom_octree {
  * @param[in] value_test boolean if tests evaluated before body execution
  * @param[in] body code to execute
  */
-#define GrGeomOctreeExteriorNodeLoop_default(i, j, k, node, octree, level,            \
-                                     ix, iy, iz, nx, ny, nz, value_test,              \
-                                     body)                                            \
-  {                                                                                   \
-    int PV_i, PV_j, PV_k, PV_l;                                                       \
-    int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                               \
-                                                                                      \
-                                                                                      \
-    PV_i = i;                                                                         \
-    PV_j = j;                                                                         \
-    PV_k = k;                                                                         \
-                                                                                      \
-    GrGeomOctreeExteriorLoop(PV_i, PV_j, PV_k, PV_l, node, octree, level, value_test, \
-    {                                                                                 \
-      if ((PV_i >= ix) && (PV_i < (ix + nx)) &&                                       \
-          (PV_j >= iy) && (PV_j < (iy + ny)) &&                                       \
-          (PV_k >= iz) && (PV_k < (iz + nz)))                                         \
-      {                                                                               \
-        i = PV_i;                                                                     \
-        j = PV_j;                                                                     \
-        k = PV_k;                                                                     \
-        body;                                                                         \
-      }                                                                               \
-    },                                                                                \
-    {                                                                                 \
-      /* find octree and region intersection */                                       \
-      PV_ixl = pfmax(ix, PV_i);                                                       \
-      PV_iyl = pfmax(iy, PV_j);                                                       \
-      PV_izl = pfmax(iz, PV_k);                                                       \
-      PV_ixu = pfmin((ix + nx), (PV_i + (int)PV_inc));                                \
-      PV_iyu = pfmin((iy + ny), (PV_j + (int)PV_inc));                                \
-      PV_izu = pfmin((iz + nz), (PV_k + (int)PV_inc));                                \
-                                                                                      \
-      /* loop over indexes and execute the body */                                    \
-      for (k = PV_izl; k < PV_izu; k++)                                               \
-        for (j = PV_iyl; j < PV_iyu; j++)                                             \
-          for (i = PV_ixl; i < PV_ixu; i++)                                           \
-          {                                                                           \
-            body;                                                                     \
-          }                                                                           \
-    })                                                                                \
-  }
+#define GrGeomOctreeExteriorNodeLoop_default(i, j, k, node, octree, level,                    \
+                                             ix, iy, iz, nx, ny, nz, value_test,              \
+                                             body)                                            \
+        {                                                                                     \
+          int PV_i, PV_j, PV_k, PV_l;                                                         \
+          int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                                 \
+                                                                                              \
+                                                                                              \
+          PV_i = i;                                                                           \
+          PV_j = j;                                                                           \
+          PV_k = k;                                                                           \
+                                                                                              \
+          GrGeomOctreeExteriorLoop(PV_i, PV_j, PV_k, PV_l, node, octree, level, value_test,   \
+    {                                                                                         \
+      if ((PV_i >= ix) && (PV_i < (ix + nx)) &&                                               \
+          (PV_j >= iy) && (PV_j < (iy + ny)) &&                                               \
+          (PV_k >= iz) && (PV_k < (iz + nz)))                                                 \
+      {                                                                                       \
+        i = PV_i;                                                                             \
+        j = PV_j;                                                                             \
+        k = PV_k;                                                                             \
+        body;                                                                                 \
+      }                                                                                       \
+    },                                                                                        \
+    {                                                                                         \
+      /* find octree and region intersection */                                               \
+      PV_ixl = pfmax(ix, PV_i);                                                               \
+      PV_iyl = pfmax(iy, PV_j);                                                               \
+      PV_izl = pfmax(iz, PV_k);                                                               \
+      PV_ixu = pfmin((ix + nx), (PV_i + (int)PV_inc));                                        \
+      PV_iyu = pfmin((iy + ny), (PV_j + (int)PV_inc));                                        \
+      PV_izu = pfmin((iz + nz), (PV_k + (int)PV_inc));                                        \
+                                                                                              \
+      /* loop over indexes and execute the body */                                            \
+      for (k = PV_izl; k < PV_izu; k++)                                                       \
+      for (j = PV_iyl; j < PV_iyu; j++)                                                       \
+      for (i = PV_ixl; i < PV_ixu; i++)                                                       \
+      {                                                                                       \
+        body;                                                                                 \
+      }                                                                                       \
+    })                                                                                        \
+        }
 
 /**
  * @brief Loop over the inside index space of an octree.
@@ -919,48 +919,48 @@ typedef struct grgeom_octree {
  * @param[in] value_test boolean if tests evaluated before body execution
  * @param[in] body code to execute
  */
-#define GrGeomOctreeInsideNodeLoop(i, j, k, node, octree, level,                    \
-                                   ix, iy, iz, nx, ny, nz, value_test,              \
-                                   body)                                            \
-  {                                                                                 \
-    int PV_i, PV_j, PV_k, PV_l;                                                     \
-    int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                             \
-                                                                                    \
-                                                                                    \
-    PV_i = i;                                                                       \
-    PV_j = j;                                                                       \
-    PV_k = k;                                                                       \
-                                                                                    \
-    GrGeomOctreeInsideLoop(PV_i, PV_j, PV_k, PV_l, node, octree, level, value_test, \
-    {                                                                               \
-      if ((PV_i >= ix) && (PV_i < (ix + nx)) &&                                     \
-          (PV_j >= iy) && (PV_j < (iy + ny)) &&                                     \
-          (PV_k >= iz) && (PV_k < (iz + nz)))                                       \
-      {                                                                             \
-        i = PV_i;                                                                   \
-        j = PV_j;                                                                   \
-        k = PV_k;                                                                   \
-        body;                                                                       \
-      }                                                                             \
-    },                                                                              \
-    {                                                                               \
-      /* find octree and region intersection */                                     \
-      PV_ixl = pfmax(ix, PV_i);                                                     \
-      PV_iyl = pfmax(iy, PV_j);                                                     \
-      PV_izl = pfmax(iz, PV_k);                                                     \
-      PV_ixu = pfmin((ix + nx), (PV_i + (int)PV_inc));                              \
-      PV_iyu = pfmin((iy + ny), (PV_j + (int)PV_inc));                              \
-      PV_izu = pfmin((iz + nz), (PV_k + (int)PV_inc));                              \
-                                                                                    \
-      /* loop over indexes and execute the body */                                  \
-      for (k = PV_izl; k < PV_izu; k++)                                             \
-        for (j = PV_iyl; j < PV_iyu; j++)                                           \
-          for (i = PV_ixl; i < PV_ixu; i++)                                         \
-          {                                                                         \
-            body;                                                                   \
-          }                                                                         \
-    })                                                                              \
-  }
+#define GrGeomOctreeInsideNodeLoop(i, j, k, node, octree, level,                          \
+                                   ix, iy, iz, nx, ny, nz, value_test,                    \
+                                   body)                                                  \
+        {                                                                                 \
+          int PV_i, PV_j, PV_k, PV_l;                                                     \
+          int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                             \
+                                                                                          \
+                                                                                          \
+          PV_i = i;                                                                       \
+          PV_j = j;                                                                       \
+          PV_k = k;                                                                       \
+                                                                                          \
+          GrGeomOctreeInsideLoop(PV_i, PV_j, PV_k, PV_l, node, octree, level, value_test, \
+    {                                                                                     \
+      if ((PV_i >= ix) && (PV_i < (ix + nx)) &&                                           \
+          (PV_j >= iy) && (PV_j < (iy + ny)) &&                                           \
+          (PV_k >= iz) && (PV_k < (iz + nz)))                                             \
+      {                                                                                   \
+        i = PV_i;                                                                         \
+        j = PV_j;                                                                         \
+        k = PV_k;                                                                         \
+        body;                                                                             \
+      }                                                                                   \
+    },                                                                                    \
+    {                                                                                     \
+      /* find octree and region intersection */                                           \
+      PV_ixl = pfmax(ix, PV_i);                                                           \
+      PV_iyl = pfmax(iy, PV_j);                                                           \
+      PV_izl = pfmax(iz, PV_k);                                                           \
+      PV_ixu = pfmin((ix + nx), (PV_i + (int)PV_inc));                                    \
+      PV_iyu = pfmin((iy + ny), (PV_j + (int)PV_inc));                                    \
+      PV_izu = pfmin((iz + nz), (PV_k + (int)PV_inc));                                    \
+                                                                                          \
+      /* loop over indexes and execute the body */                                        \
+      for (k = PV_izl; k < PV_izu; k++)                                                   \
+      for (j = PV_iyl; j < PV_iyu; j++)                                                   \
+      for (i = PV_ixl; i < PV_ixu; i++)                                                   \
+      {                                                                                   \
+        body;                                                                             \
+      }                                                                                   \
+    })                                                                                    \
+        }
 
 /**
  * @brief Loop over the index space of an octree with strides.
@@ -986,66 +986,66 @@ typedef struct grgeom_octree {
  * @param[in] value_test boolean if tests evaluated before body execution
  * @param[in] body code to execute for leaf nodes or nodes at the specified level
  */
-#define GrGeomOctreeNodeLoop2(i, j, k, node, octree, level,                   \
-                              ix, iy, iz, nx, ny, nz, sx, sy, sz, value_test, \
-                              body)                                           \
-  {                                                                           \
-    int PV_i, PV_j, PV_k, PV_l;                                               \
-    int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                       \
-                                                                              \
-                                                                              \
-    PV_i = i;                                                                 \
-    PV_j = j;                                                                 \
-    PV_k = k;                                                                 \
-                                                                              \
-    GrGeomOctreeLoop(PV_i, PV_j, PV_k, PV_l, node, octree, level, value_test, \
-    {                                                                         \
-      if ((PV_i >= ix) && (PV_i < (ix + nx)) &&                               \
-          (PV_j >= iy) && (PV_j < (iy + ny)) &&                               \
-          (PV_k >= iz) && (PV_k < (iz + nz)) &&                               \
-          ((PV_i - ix) % sx == 0) &&                                          \
-          ((PV_j - iy) % sy == 0) &&                                          \
-          ((PV_k - iz) % sz == 0))                                            \
-      {                                                                       \
-        i = PV_i;                                                             \
-        j = PV_j;                                                             \
-        k = PV_k;                                                             \
-                                                                              \
-        body;                                                                 \
-      }                                                                       \
-    },                                                                        \
-    {                                                                         \
-      /* find octree and region intersection */                               \
-      PV_ixl = pfmax(ix, PV_i);                                               \
-      PV_iyl = pfmax(iy, PV_j);                                               \
-      PV_izl = pfmax(iz, PV_k);                                               \
-      PV_ixu = pfmin((ix + nx), (PV_i + (int)PV_inc));                        \
-      PV_iyu = pfmin((iy + ny), (PV_j + (int)PV_inc));                        \
-      PV_izu = pfmin((iz + nz), (PV_k + (int)PV_inc));                        \
-                                                                              \
-      /* project intersection onto strided index space */                     \
-      PV_ixl = PV_ixl + ix;                                                   \
-      PV_ixu = PV_ixu + ix;                                                   \
-      PV_ixl = ((int)((PV_ixl + (sx - 1)) / sx)) * sx - ix;                   \
-      PV_ixu = ((int)((PV_ixu + (sx - 1)) / sx)) * sx - ix;                   \
-      PV_iyl = PV_iyl + iy;                                                   \
-      PV_iyu = PV_iyu + iy;                                                   \
-      PV_iyl = ((int)((PV_iyl + (sy - 1)) / sy)) * sy - iy;                   \
-      PV_iyu = ((int)((PV_iyu + (sy - 1)) / sy)) * sy - iy;                   \
-      PV_izl = PV_izl + iz;                                                   \
-      PV_izu = PV_izu + iz;                                                   \
-      PV_izl = ((int)((PV_izl + (sz - 1)) / sz)) * sz - iz;                   \
-      PV_izu = ((int)((PV_izu + (sz - 1)) / sz)) * sz - iz;                   \
-                                                                              \
-      /* loop over indexes and execute the body */                            \
-      for (k = PV_izl; k < PV_izu; k += sz)                                   \
-        for (j = PV_iyl; j < PV_iyu; j += sy)                                 \
-          for (i = PV_ixl; i < PV_ixu; i += sx)                               \
-          {                                                                   \
-            body;                                                             \
-          }                                                                   \
-    })                                                                        \
-  }
+#define GrGeomOctreeNodeLoop2(i, j, k, node, octree, level,                         \
+                              ix, iy, iz, nx, ny, nz, sx, sy, sz, value_test,       \
+                              body)                                                 \
+        {                                                                           \
+          int PV_i, PV_j, PV_k, PV_l;                                               \
+          int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                       \
+                                                                                    \
+                                                                                    \
+          PV_i = i;                                                                 \
+          PV_j = j;                                                                 \
+          PV_k = k;                                                                 \
+                                                                                    \
+          GrGeomOctreeLoop(PV_i, PV_j, PV_k, PV_l, node, octree, level, value_test, \
+    {                                                                               \
+      if ((PV_i >= ix) && (PV_i < (ix + nx)) &&                                     \
+          (PV_j >= iy) && (PV_j < (iy + ny)) &&                                     \
+          (PV_k >= iz) && (PV_k < (iz + nz)) &&                                     \
+          ((PV_i - ix) % sx == 0) &&                                                \
+          ((PV_j - iy) % sy == 0) &&                                                \
+          ((PV_k - iz) % sz == 0))                                                  \
+      {                                                                             \
+        i = PV_i;                                                                   \
+        j = PV_j;                                                                   \
+        k = PV_k;                                                                   \
+                                                                                    \
+        body;                                                                       \
+      }                                                                             \
+    },                                                                              \
+    {                                                                               \
+      /* find octree and region intersection */                                     \
+      PV_ixl = pfmax(ix, PV_i);                                                     \
+      PV_iyl = pfmax(iy, PV_j);                                                     \
+      PV_izl = pfmax(iz, PV_k);                                                     \
+      PV_ixu = pfmin((ix + nx), (PV_i + (int)PV_inc));                              \
+      PV_iyu = pfmin((iy + ny), (PV_j + (int)PV_inc));                              \
+      PV_izu = pfmin((iz + nz), (PV_k + (int)PV_inc));                              \
+                                                                                    \
+      /* project intersection onto strided index space */                           \
+      PV_ixl = PV_ixl + ix;                                                         \
+      PV_ixu = PV_ixu + ix;                                                         \
+      PV_ixl = ((int)((PV_ixl + (sx - 1)) / sx)) * sx - ix;                         \
+      PV_ixu = ((int)((PV_ixu + (sx - 1)) / sx)) * sx - ix;                         \
+      PV_iyl = PV_iyl + iy;                                                         \
+      PV_iyu = PV_iyu + iy;                                                         \
+      PV_iyl = ((int)((PV_iyl + (sy - 1)) / sy)) * sy - iy;                         \
+      PV_iyu = ((int)((PV_iyu + (sy - 1)) / sy)) * sy - iy;                         \
+      PV_izl = PV_izl + iz;                                                         \
+      PV_izu = PV_izu + iz;                                                         \
+      PV_izl = ((int)((PV_izl + (sz - 1)) / sz)) * sz - iz;                         \
+      PV_izu = ((int)((PV_izu + (sz - 1)) / sz)) * sz - iz;                         \
+                                                                                    \
+      /* loop over indexes and execute the body */                                  \
+      for (k = PV_izl; k < PV_izu; k += sz)                                         \
+      for (j = PV_iyl; j < PV_iyu; j += sy)                                         \
+      for (i = PV_ixl; i < PV_ixu; i += sx)                                         \
+      {                                                                             \
+        body;                                                                       \
+      }                                                                             \
+    })                                                                              \
+        }
 
 // \todo SGS 2019/04/18 Currently faces are stored on single point leaf nodes.
 // Would be better to store planes.
@@ -1101,88 +1101,88 @@ typedef struct grgeom_octree {
  * @param[in] nz z upper bound of index space to process
  * @param[in] body code to execute
  */
-#define GrGeomOctreeFaceLoop(i, j, k, fdir, node, octree, level_of_interest, \
-                             ix, iy, iz, nx, ny, nz, body)                   \
-  {                                                                          \
-    int PV_f;                                                                \
-    int PV_fdir[3];                                                          \
-                                                                             \
-                                                                             \
-    fdir = PV_fdir;                                                          \
-    GrGeomOctreeInsideNodeLoop(i, j, k, node, octree, level_of_interest,     \
-                               ix, iy, iz, nx, ny, nz,                       \
-                               TRUE,                                         \
-    {                                                                        \
-      for (PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                    \
-        if (GrGeomOctreeHasFace(node, PV_f))                                 \
-        {                                                                    \
-          switch (PV_f)                                                      \
-          {                                                                  \
-            case GrGeomOctreeFaceL:                                          \
-              fdir[0] = -1; fdir[1] = 0; fdir[2] = 0;                        \
-              break;                                                         \
-            case GrGeomOctreeFaceR:                                          \
-              fdir[0] = 1; fdir[1] = 0; fdir[2] = 0;                         \
-              break;                                                         \
-            case GrGeomOctreeFaceD:                                          \
-              fdir[0] = 0; fdir[1] = -1; fdir[2] = 0;                        \
-              break;                                                         \
-            case GrGeomOctreeFaceU:                                          \
-              fdir[0] = 0; fdir[1] = 1; fdir[2] = 0;                         \
-              break;                                                         \
-            case GrGeomOctreeFaceB:                                          \
-              fdir[0] = 0; fdir[1] = 0; fdir[2] = -1;                        \
-              break;                                                         \
-            case GrGeomOctreeFaceF:                                          \
-              fdir[0] = 0; fdir[1] = 0; fdir[2] = 1;                         \
-              break;                                                         \
-            default:                                                         \
-              fdir[0] = -9999; fdir[1] = -9999; fdir[2] = -99999;            \
-              break;                                                         \
-          }                                                                  \
-                                                                             \
-          body;                                                              \
-        }                                                                    \
-    })                                                                       \
-  }
+#define GrGeomOctreeFaceLoop(i, j, k, fdir, node, octree, level_of_interest,       \
+                             ix, iy, iz, nx, ny, nz, body)                         \
+        {                                                                          \
+          int PV_f;                                                                \
+          int PV_fdir[3];                                                          \
+                                                                                   \
+                                                                                   \
+          fdir = PV_fdir;                                                          \
+          GrGeomOctreeInsideNodeLoop(i, j, k, node, octree, level_of_interest,     \
+                                     ix, iy, iz, nx, ny, nz,                       \
+                                     TRUE,                                         \
+    {                                                                              \
+      for (PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                          \
+      if (GrGeomOctreeHasFace(node, PV_f))                                         \
+      {                                                                            \
+        switch (PV_f)                                                              \
+        {                                                                          \
+            case GrGeomOctreeFaceL:                                                \
+              fdir[0] = -1; fdir[1] = 0; fdir[2] = 0;                              \
+              break;                                                               \
+            case GrGeomOctreeFaceR:                                                \
+              fdir[0] = 1; fdir[1] = 0; fdir[2] = 0;                               \
+              break;                                                               \
+            case GrGeomOctreeFaceD:                                                \
+              fdir[0] = 0; fdir[1] = -1; fdir[2] = 0;                              \
+              break;                                                               \
+            case GrGeomOctreeFaceU:                                                \
+              fdir[0] = 0; fdir[1] = 1; fdir[2] = 0;                               \
+              break;                                                               \
+            case GrGeomOctreeFaceB:                                                \
+              fdir[0] = 0; fdir[1] = 0; fdir[2] = -1;                              \
+              break;                                                               \
+            case GrGeomOctreeFaceF:                                                \
+              fdir[0] = 0; fdir[1] = 0; fdir[2] = 1;                               \
+              break;                                                               \
+            default:                                                               \
+              fdir[0] = -9999; fdir[1] = -9999; fdir[2] = -99999;                  \
+              break;                                                               \
+        }                                                                          \
+                                                                                   \
+        body;                                                                      \
+      }                                                                            \
+    })                                                                             \
+        }
 
 /**
  * @brief Loop over the faces of an octree without use of fdir array
  */
-#define GrGeomOctreeFaceLoopNoFdir(i, j, k,                             \
-                                   node, octree, level_of_interest,     \
-                                   ix, iy, iz, nx, ny, nz,              \
-                                   locals, setup,                       \
-                                   f_left, f_right,                     \
-                                   f_down, f_up,                        \
-                                   f_back, f_front,                     \
-                                   finalize)                            \
-  {                                                                     \
-    int PV_f;                                                           \
-    UNPACK(locals);                                                     \
-    GrGeomOctreeInsideNodeLoop(i, j, k, node, octree, level_of_interest, \
-                               ix, iy, iz, nx, ny, nz,                  \
-                               TRUE,                                    \
-    {                                                                   \
-      for (PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)               \
-        if (GrGeomOctreeHasFace(node, PV_f))                            \
-        {                                                               \
-          setup;                                                        \
-                                                                        \
-          switch (PV_f)                                                 \
-          {                                                             \
-            f_left;                                                     \
-            f_right;                                                    \
-            f_down;                                                     \
-            f_up;                                                       \
-            f_back;                                                     \
-            f_front;                                                    \
-          }                                                             \
-                                                                        \
-          finalize;                                                     \
-        }                                                               \
-    })                                                                  \
-  }
+#define GrGeomOctreeFaceLoopNoFdir(i, j, k,                                    \
+                                   node, octree, level_of_interest,            \
+                                   ix, iy, iz, nx, ny, nz,                     \
+                                   locals, setup,                              \
+                                   f_left, f_right,                            \
+                                   f_down, f_up,                               \
+                                   f_back, f_front,                            \
+                                   finalize)                                   \
+        {                                                                      \
+          int PV_f;                                                            \
+          UNPACK(locals);                                                      \
+          GrGeomOctreeInsideNodeLoop(i, j, k, node, octree, level_of_interest, \
+                                     ix, iy, iz, nx, ny, nz,                   \
+                                     TRUE,                                     \
+    {                                                                          \
+      for (PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                      \
+      if (GrGeomOctreeHasFace(node, PV_f))                                     \
+      {                                                                        \
+        setup;                                                                 \
+                                                                               \
+        switch (PV_f)                                                          \
+        {                                                                      \
+          f_left;                                                              \
+          f_right;                                                             \
+          f_down;                                                              \
+          f_up;                                                                \
+          f_back;                                                              \
+          f_front;                                                             \
+        }                                                                      \
+                                                                               \
+        finalize;                                                              \
+      }                                                                        \
+    })                                                                         \
+        }
 
 /*==========================================================================
  *==========================================================================*/
@@ -1220,75 +1220,75 @@ typedef struct grgeom_octree {
  * @param[in] level_body code to execute if level has been reached.
  * @param[in] leaf_body code to execute for leaf nodes
  */
-#define GrGeomOctreeBoxLoop(i, j, k, l, node,              \
-                            octree, levels_in_octree,      \
-                            level_of_interest,             \
-                            value_test,                    \
-                            level_body, leaf_body)         \
-  {                                                        \
-    unsigned int PV_inc;                                   \
-    int           *PV_visiting;                            \
-    int PV_visit_child;                                    \
-                                                           \
-    node = octree;                                         \
-    l = 0;                                                 \
-                                                           \
-    PV_inc = 1 << (levels_in_octree);                      \
-    PV_visiting = ctalloc(int, (levels_in_octree) + 2);    \
-    PV_visiting++;                                         \
-    PV_visiting[0] = 0;                                    \
-                                                           \
-    while (l >= 0)                                         \
-    {                                                      \
-      /* if at the level of interest */                    \
-      if (l == (level_of_interest))                        \
-      {                                                    \
-        if (value_test)                                    \
-          level_body;                                      \
-                                                           \
-        PV_visit_child = FALSE;                            \
-      }                                                    \
-                                                           \
-      /* if this is a leaf node */                         \
-      else if (GrGeomOctreeNodeIsLeaf(node))               \
-      {                                                    \
-        if (value_test)                                    \
-          leaf_body;                                       \
-                                                           \
-        PV_visit_child = FALSE;                            \
-      }                                                    \
-                                                           \
-      /* have I visited all of the children? */            \
-      else if (PV_visiting[l] < GrGeomOctreeNumChildren)   \
-        PV_visit_child = TRUE;                             \
-      else                                                 \
-        PV_visit_child = FALSE;                            \
-                                                           \
-      /* visit either a child or the parent node */        \
-      if (PV_visit_child)                                  \
-      {                                                    \
-        node = GrGeomOctreeChild(node, PV_visiting[l]);    \
-        PV_inc = PV_inc >> 1;                              \
-        i += (int)PV_inc * ((PV_visiting[l] & 1) ? 1 : 0); \
-        j += (int)PV_inc * ((PV_visiting[l] & 2) ? 1 : 0); \
-        k += (int)PV_inc * ((PV_visiting[l] & 4) ? 1 : 0); \
-        l++;                                               \
-        PV_visiting[l] = 0;                                \
-      }                                                    \
-      else                                                 \
-      {                                                    \
-        l--;                                               \
-        i -= (int)PV_inc * ((PV_visiting[l] & 1) ? 1 : 0); \
-        j -= (int)PV_inc * ((PV_visiting[l] & 2) ? 1 : 0); \
-        k -= (int)PV_inc * ((PV_visiting[l] & 4) ? 1 : 0); \
-        PV_inc = PV_inc << 1;                              \
-        node = GrGeomOctreeParent(node);                   \
-        PV_visiting[l]++;                                  \
-      }                                                    \
-    }                                                      \
-                                                           \
-    tfree(PV_visiting - 1);                                \
-  }
+#define GrGeomOctreeBoxLoop(i, j, k, l, node,                    \
+                            octree, levels_in_octree,            \
+                            level_of_interest,                   \
+                            value_test,                          \
+                            level_body, leaf_body)               \
+        {                                                        \
+          unsigned int PV_inc;                                   \
+          int           *PV_visiting;                            \
+          int PV_visit_child;                                    \
+                                                                 \
+          node = octree;                                         \
+          l = 0;                                                 \
+                                                                 \
+          PV_inc = 1 << (levels_in_octree);                      \
+          PV_visiting = ctalloc(int, (levels_in_octree) + 2);    \
+          PV_visiting++;                                         \
+          PV_visiting[0] = 0;                                    \
+                                                                 \
+          while (l >= 0)                                         \
+          {                                                      \
+            /* if at the level of interest */                    \
+            if (l == (level_of_interest))                        \
+            {                                                    \
+              if (value_test)                                    \
+              level_body;                                        \
+                                                                 \
+              PV_visit_child = FALSE;                            \
+            }                                                    \
+                                                                 \
+            /* if this is a leaf node */                         \
+            else if (GrGeomOctreeNodeIsLeaf(node))               \
+            {                                                    \
+              if (value_test)                                    \
+              leaf_body;                                         \
+                                                                 \
+              PV_visit_child = FALSE;                            \
+            }                                                    \
+                                                                 \
+            /* have I visited all of the children? */            \
+            else if (PV_visiting[l] < GrGeomOctreeNumChildren)   \
+            PV_visit_child = TRUE;                               \
+            else                                                 \
+            PV_visit_child = FALSE;                              \
+                                                                 \
+            /* visit either a child or the parent node */        \
+            if (PV_visit_child)                                  \
+            {                                                    \
+              node = GrGeomOctreeChild(node, PV_visiting[l]);    \
+              PV_inc = PV_inc >> 1;                              \
+              i += (int)PV_inc * ((PV_visiting[l] & 1) ? 1 : 0); \
+              j += (int)PV_inc * ((PV_visiting[l] & 2) ? 1 : 0); \
+              k += (int)PV_inc * ((PV_visiting[l] & 4) ? 1 : 0); \
+              l++;                                               \
+              PV_visiting[l] = 0;                                \
+            }                                                    \
+            else                                                 \
+            {                                                    \
+              l--;                                               \
+              i -= (int)PV_inc * ((PV_visiting[l] & 1) ? 1 : 0); \
+              j -= (int)PV_inc * ((PV_visiting[l] & 2) ? 1 : 0); \
+              k -= (int)PV_inc * ((PV_visiting[l] & 4) ? 1 : 0); \
+              PV_inc = PV_inc << 1;                              \
+              node = GrGeomOctreeParent(node);                   \
+              PV_visiting[l]++;                                  \
+            }                                                    \
+          }                                                      \
+                                                                 \
+          tfree(PV_visiting - 1);                                \
+        }
 
 
 /**
@@ -1327,64 +1327,64 @@ typedef struct grgeom_octree {
  * @param[in] value_test boolean if tests evaluated before body execution
  * @param[in] body code to execute
  */
-#define GrGeomOctreeNodeBoxLoop(i, j, k,                            \
-                                num_i, num_j, num_k,                \
-                                node, octree,                       \
-                                levels_in_octree,                   \
-                                level_of_interest,                  \
-                                ix, iy, iz, nx, ny, nz, value_test, \
-                                body)                               \
-  {                                                                 \
-    int PV_i, PV_j, PV_k, PV_l;                                     \
-    int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;             \
-                                                                    \
-                                                                    \
-    PV_i = i;                                                       \
-    PV_j = j;                                                       \
-    PV_k = k;                                                       \
-                                                                    \
-    GrGeomOctreeBoxLoop(PV_i, PV_j, PV_k, PV_l,                     \
-                        node, octree, levels_in_octree,             \
-                        level_of_interest, value_test,              \
-    {                                                               \
-      /* find octree and region intersection */                     \
-      PV_ixl = pfmax(ix, PV_i);                                     \
-      PV_iyl = pfmax(iy, PV_j);                                     \
-      PV_izl = pfmax(iz, PV_k);                                     \
-      PV_ixu = pfmin((ix + nx), (PV_i + (int)PV_inc));              \
-      PV_iyu = pfmin((iy + ny), (PV_j + (int)PV_inc));              \
-      PV_izu = pfmin((iz + nz), (PV_k + (int)PV_inc));              \
-                                                                    \
-      i = PV_ixl;                                                   \
-      j = PV_iyl;                                                   \
-      k = PV_izl;                                                   \
-      num_i = PV_ixu - PV_ixl;                                      \
-      num_j = PV_iyu - PV_iyl;                                      \
-      num_k = PV_izu - PV_izl;                                      \
-      if (num_i > 0 && num_j > 0 && num_k > 0) {                    \
-        body;                                                       \
-      }                                                             \
-    },                                                              \
-    {                                                               \
-      /* find octree and region intersection */                     \
-      PV_ixl = pfmax(ix, PV_i);                                     \
-      PV_iyl = pfmax(iy, PV_j);                                     \
-      PV_izl = pfmax(iz, PV_k);                                     \
-      PV_ixu = pfmin((ix + nx), (PV_i + (int)PV_inc));              \
-      PV_iyu = pfmin((iy + ny), (PV_j + (int)PV_inc));              \
-      PV_izu = pfmin((iz + nz), (PV_k + (int)PV_inc));              \
-                                                                    \
-      i = PV_ixl;                                                   \
-      j = PV_iyl;                                                   \
-      k = PV_izl;                                                   \
-      num_i = PV_ixu - PV_ixl;                                      \
-      num_j = PV_iyu - PV_iyl;                                      \
-      num_k = PV_izu - PV_izl;                                      \
-      if (num_i > 0 && num_j > 0 && num_k > 0) {                    \
-        body;                                                       \
-      }                                                             \
-    })                                                              \
-  }
+#define GrGeomOctreeNodeBoxLoop(i, j, k,                                  \
+                                num_i, num_j, num_k,                      \
+                                node, octree,                             \
+                                levels_in_octree,                         \
+                                level_of_interest,                        \
+                                ix, iy, iz, nx, ny, nz, value_test,       \
+                                body)                                     \
+        {                                                                 \
+          int PV_i, PV_j, PV_k, PV_l;                                     \
+          int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;             \
+                                                                          \
+                                                                          \
+          PV_i = i;                                                       \
+          PV_j = j;                                                       \
+          PV_k = k;                                                       \
+                                                                          \
+          GrGeomOctreeBoxLoop(PV_i, PV_j, PV_k, PV_l,                     \
+                              node, octree, levels_in_octree,             \
+                              level_of_interest, value_test,              \
+    {                                                                     \
+      /* find octree and region intersection */                           \
+      PV_ixl = pfmax(ix, PV_i);                                           \
+      PV_iyl = pfmax(iy, PV_j);                                           \
+      PV_izl = pfmax(iz, PV_k);                                           \
+      PV_ixu = pfmin((ix + nx), (PV_i + (int)PV_inc));                    \
+      PV_iyu = pfmin((iy + ny), (PV_j + (int)PV_inc));                    \
+      PV_izu = pfmin((iz + nz), (PV_k + (int)PV_inc));                    \
+                                                                          \
+      i = PV_ixl;                                                         \
+      j = PV_iyl;                                                         \
+      k = PV_izl;                                                         \
+      num_i = PV_ixu - PV_ixl;                                            \
+      num_j = PV_iyu - PV_iyl;                                            \
+      num_k = PV_izu - PV_izl;                                            \
+      if (num_i > 0 && num_j > 0 && num_k > 0) {                          \
+        body;                                                             \
+      }                                                                   \
+    },                                                                    \
+    {                                                                     \
+      /* find octree and region intersection */                           \
+      PV_ixl = pfmax(ix, PV_i);                                           \
+      PV_iyl = pfmax(iy, PV_j);                                           \
+      PV_izl = pfmax(iz, PV_k);                                           \
+      PV_ixu = pfmin((ix + nx), (PV_i + (int)PV_inc));                    \
+      PV_iyu = pfmin((iy + ny), (PV_j + (int)PV_inc));                    \
+      PV_izu = pfmin((iz + nz), (PV_k + (int)PV_inc));                    \
+                                                                          \
+      i = PV_ixl;                                                         \
+      j = PV_iyl;                                                         \
+      k = PV_izl;                                                         \
+      num_i = PV_ixu - PV_ixl;                                            \
+      num_j = PV_iyu - PV_iyl;                                            \
+      num_k = PV_izu - PV_izl;                                            \
+      if (num_i > 0 && num_j > 0 && num_k > 0) {                          \
+        body;                                                             \
+      }                                                                   \
+    })                                                                    \
+        }
 
 
 #endif
diff --git a/pfsimulator/parflow_lib/grgeometry.c b/pfsimulator/parflow_lib/grgeometry.c
index 59baca0ca..f2e40f0ad 100644
--- a/pfsimulator/parflow_lib/grgeometry.c
+++ b/pfsimulator/parflow_lib/grgeometry.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Member functions for the GrGeometry (Grid dependent Geometry) class.
@@ -296,14 +296,14 @@ GrGeomSolid   *GrGeomNewSolid(
   new_grgeomsolid->interior_boxes = NULL;
 
 #if defined(PARFLOW_HAVE_CUDA) || defined(PARFLOW_HAVE_KOKKOS)
-  GrGeomSolidCellFlagData(new_grgeomsolid) = NULL; 
+  GrGeomSolidCellFlagData(new_grgeomsolid) = NULL;
   GrGeomSolidCellFlagDataSize(new_grgeomsolid) = 0;
   GrGeomSolidCellFlagInitialized(new_grgeomsolid) = 0;
 
   (new_grgeomsolid->ival) = talloc(int**, GrGeomOctreeNumFaces);
   for (int f = 0; f < GrGeomOctreeNumFaces; f++)
-  { 
-    (new_grgeomsolid->ival[f]) = talloc(int*, 2 * num_patches);    
+  {
+    (new_grgeomsolid->ival[f]) = talloc(int*, 2 * num_patches);
     for (int ipatch = 0; ipatch < 2 * num_patches; ipatch++)
       (new_grgeomsolid->ival[f][ipatch]) = NULL;
   }
@@ -363,14 +363,15 @@ void          GrGeomFreeSolid(
 
 #if defined(PARFLOW_HAVE_CUDA) || defined(PARFLOW_HAVE_KOKKOS)
   // Internal _tfree_device function is used because unified memory is not active in this comp unit
-  if(GrGeomSolidCellFlagData(solid)) _tfree_device(GrGeomSolidCellFlagData(solid));
+  if (GrGeomSolidCellFlagData(solid))
+    _tfree_device(GrGeomSolidCellFlagData(solid));
 
   for (int f = 0; f < GrGeomOctreeNumFaces; f++)
   {
     for (int ipatch = 0; ipatch < 2 * GrGeomSolidNumPatches(solid); ipatch++)
     {
       int *ival = GrGeomSolidCellIval(solid, ipatch, f);
-      if(ival)
+      if (ival)
         _tfree_device(ival);
     }
     tfree(solid->ival[f]);
diff --git a/pfsimulator/parflow_lib/grgeometry.h b/pfsimulator/parflow_lib/grgeometry.h
index 5a665ce79..d97f222fc 100644
--- a/pfsimulator/parflow_lib/grgeometry.h
+++ b/pfsimulator/parflow_lib/grgeometry.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Geometry class structures and accessors
 *
@@ -66,9 +66,9 @@ typedef struct {
    *  Bits 2-7: GrGeomSurfLoop (1 bit for each face)
    */
 
-  /* A pointer to an array for each relevant cell. 
-   * Bits 0-7 of each element determine if the cell is evaluated by the respective loop. 
-   */ 
+  /* A pointer to an array for each relevant cell.
+   * Bits 0-7 of each element determine if the cell is evaluated by the respective loop.
+   */
   char *cell_mask;
 
   /* The size of cell_mask array in bytes. */
@@ -77,7 +77,7 @@ typedef struct {
   /* Bits 0-7 indicate if cell_mask has been filled (ie. initialized) for the respective loop */
   char loops_masked;
 
-  /* ival indices for parallel exec of patchloops */ 
+  /* ival indices for parallel exec of patchloops */
   int ***ival;
 #endif
 
@@ -133,55 +133,55 @@ typedef struct {
  *--------------------------------------------------------------------------*/
 
 #define GrGeomInLoopBoxes_default(i, j, k, grgeom, ix, iy, iz, nx, ny, nz, body) \
-  {                                                                      \
-    int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                  \
-    int *PV_visiting = NULL;                                             \
-    PF_UNUSED(PV_visiting);                                              \
-    BoxArray* boxes = GrGeomSolidInteriorBoxes(grgeom);                  \
-    for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)         \
-    {                                                                    \
-      Box box = BoxArrayGetBox(boxes, PV_box);                           \
-      /* find octree and region intersection */                          \
-      PV_ixl = pfmax(ix, box.lo[0]);                                     \
-      PV_iyl = pfmax(iy, box.lo[1]);                                     \
-      PV_izl = pfmax(iz, box.lo[2]);                                     \
-      PV_ixu = pfmin((ix + nx - 1), box.up[0]);                          \
-      PV_iyu = pfmin((iy + ny - 1), box.up[1]);                          \
-      PV_izu = pfmin((iz + nz - 1), box.up[2]);                          \
-                                                                         \
-      for (k = PV_izl; k <= PV_izu; k++)                                 \
-        for (j = PV_iyl; j <= PV_iyu; j++)                               \
-          for (i = PV_ixl; i <= PV_ixu; i++)                             \
-          {                                                              \
-            body;                                                        \
-          }                                                              \
-    }                                                                    \
-  }
-
-#define GrGeomInLoop(i, j, k, grgeom,                                    \
-                     r, ix, iy, iz, nx, ny, nz, body)                    \
-  {                                                                      \
-    if (r == 0 && GrGeomSolidInteriorBoxes(grgeom))                      \
-    {                                                                    \
-      GrGeomInLoopBoxes(i, j, k, grgeom,                                 \
-                        ix, iy, iz, nx, ny, nz, body);                   \
-    }                                                                    \
-    else                                                                 \
-    {                                                                    \
-      GrGeomOctree  *PV_node;                                            \
-      double PV_ref = pow(2.0, r);                                       \
-                                                                         \
-      i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;                     \
-      j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;                     \
-      k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;                     \
-      GrGeomOctreeInteriorNodeLoop(i, j, k, PV_node,                     \
-                                   GrGeomSolidData(grgeom),              \
-                                   GrGeomSolidOctreeBGLevel(grgeom) + r, \
-                                   ix, iy, iz, nx, ny, nz,               \
-                                   TRUE,                                 \
-                                   body);                                \
-    }                                                                    \
-  }
+        {                                                                        \
+          int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                    \
+          int *PV_visiting = NULL;                                               \
+          PF_UNUSED(PV_visiting);                                                \
+          BoxArray* boxes = GrGeomSolidInteriorBoxes(grgeom);                    \
+          for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)           \
+          {                                                                      \
+            Box box = BoxArrayGetBox(boxes, PV_box);                             \
+            /* find octree and region intersection */                            \
+            PV_ixl = pfmax(ix, box.lo[0]);                                       \
+            PV_iyl = pfmax(iy, box.lo[1]);                                       \
+            PV_izl = pfmax(iz, box.lo[2]);                                       \
+            PV_ixu = pfmin((ix + nx - 1), box.up[0]);                            \
+            PV_iyu = pfmin((iy + ny - 1), box.up[1]);                            \
+            PV_izu = pfmin((iz + nz - 1), box.up[2]);                            \
+                                                                                 \
+            for (k = PV_izl; k <= PV_izu; k++)                                   \
+            for (j = PV_iyl; j <= PV_iyu; j++)                                   \
+            for (i = PV_ixl; i <= PV_ixu; i++)                                   \
+            {                                                                    \
+              body;                                                              \
+            }                                                                    \
+          }                                                                      \
+        }
+
+#define GrGeomInLoop(i, j, k, grgeom,                                          \
+                     r, ix, iy, iz, nx, ny, nz, body)                          \
+        {                                                                      \
+          if (r == 0 && GrGeomSolidInteriorBoxes(grgeom))                      \
+          {                                                                    \
+            GrGeomInLoopBoxes(i, j, k, grgeom,                                 \
+                              ix, iy, iz, nx, ny, nz, body);                   \
+          }                                                                    \
+          else                                                                 \
+          {                                                                    \
+            GrGeomOctree  *PV_node;                                            \
+            double PV_ref = pow(2.0, r);                                       \
+                                                                               \
+            i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;                     \
+            j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;                     \
+            k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;                     \
+            GrGeomOctreeInteriorNodeLoop(i, j, k, PV_node,                     \
+                                         GrGeomSolidData(grgeom),              \
+                                         GrGeomSolidOctreeBGLevel(grgeom) + r, \
+                                         ix, iy, iz, nx, ny, nz,               \
+                                         TRUE,                                 \
+                                         body);                                \
+          }                                                                    \
+        }
 
 /*--------------------------------------------------------------------------
  * GrGeomSolid looping macro:
@@ -192,24 +192,24 @@ typedef struct {
  *  Interior version of this would improve speed; but this loop is not
  * currently used.
  */
-#define GrGeomInLoop2(i, j, k, grgeom,                             \
-                      r, ix, iy, iz, nx, ny, nz, sx, sy, sz, body) \
-  {                                                                \
-    GrGeomOctree  *PV_node;                                        \
-    double PV_ref = pow(2.0, r);                                   \
-                                                                   \
-                                                                   \
-    i = GrGeomSolidOctreeIX(grgeom) * PV_ref;                      \
-    j = GrGeomSolidOctreeIY(grgeom) * PV_ref;                      \
-    k = GrGeomSolidOctreeIZ(grgeom) * PV_ref;                      \
-    GrGeomOctreeNodeLoop2(i, j, k, PV_node,                        \
-                          GrGeomSolidData(grgeom),                 \
-                          GrGeomSolidOctreeBGLevel(grgeom) + r,    \
-                          ix, iy, iz, nx, ny, nz, sx, sy, sz,      \
-                          (GrGeomOctreeNodeIsInside(PV_node) ||    \
-                           GrGeomOctreeNodeIsFull(PV_node)),       \
-                          body);                                   \
-  }
+#define GrGeomInLoop2(i, j, k, grgeom,                                   \
+                      r, ix, iy, iz, nx, ny, nz, sx, sy, sz, body)       \
+        {                                                                \
+          GrGeomOctree  *PV_node;                                        \
+          double PV_ref = pow(2.0, r);                                   \
+                                                                         \
+                                                                         \
+          i = GrGeomSolidOctreeIX(grgeom) * PV_ref;                      \
+          j = GrGeomSolidOctreeIY(grgeom) * PV_ref;                      \
+          k = GrGeomSolidOctreeIZ(grgeom) * PV_ref;                      \
+          GrGeomOctreeNodeLoop2(i, j, k, PV_node,                        \
+                                GrGeomSolidData(grgeom),                 \
+                                GrGeomSolidOctreeBGLevel(grgeom) + r,    \
+                                ix, iy, iz, nx, ny, nz, sx, sy, sz,      \
+                                (GrGeomOctreeNodeIsInside(PV_node) ||    \
+                                 GrGeomOctreeNodeIsFull(PV_node)),       \
+                                body);                                   \
+        }
 
 /*--------------------------------------------------------------------------
  * GrGeomSolid looping macro:
@@ -217,66 +217,66 @@ typedef struct {
  *--------------------------------------------------------------------------*/
 
 #define GrGeomOutLoop_default(i, j, k, grgeom,                                 \
-                      r, ix, iy, iz, nx, ny, nz, body)                 \
-  {                                                                    \
-    GrGeomOctree  *PV_node;                                            \
-    double PV_ref = pow(2.0, r);                                       \
-                                                                       \
-                                                                       \
-    i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;                     \
-    j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;                     \
-    k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;                     \
-    GrGeomOctreeExteriorNodeLoop(i, j, k, PV_node,                     \
-                                 GrGeomSolidData(grgeom),              \
-                                 GrGeomSolidOctreeBGLevel(grgeom) + r, \
-                                 ix, iy, iz, nx, ny, nz,               \
-                                 TRUE,                                 \
-                                 body);                                \
-  }
+                              r, ix, iy, iz, nx, ny, nz, body)                 \
+        {                                                                      \
+          GrGeomOctree  *PV_node;                                              \
+          double PV_ref = pow(2.0, r);                                         \
+                                                                               \
+                                                                               \
+          i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;                       \
+          j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;                       \
+          k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;                       \
+          GrGeomOctreeExteriorNodeLoop(i, j, k, PV_node,                       \
+                                       GrGeomSolidData(grgeom),                \
+                                       GrGeomSolidOctreeBGLevel(grgeom) + r,   \
+                                       ix, iy, iz, nx, ny, nz,                 \
+                                       TRUE,                                   \
+                                       body);                                  \
+        }
 
 /*--------------------------------------------------------------------------
  * GrGeomSolid looping macro:
  *   Macro for looping over the outside of a solid with non-unitary strides.
  *--------------------------------------------------------------------------*/
 
-#define GrGeomOutLoop2(i, j, k, grgeom,                             \
-                       r, ix, iy, iz, nx, ny, nz, sx, sy, sz, body) \
-  {                                                                 \
-    GrGeomOctree  *PV_node;                                         \
-    double PV_ref = pow(2.0, r);                                    \
-                                                                    \
-                                                                    \
-    i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;                  \
-    j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;                  \
-    k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;                  \
-    GrGeomOctreeNodeLoop2(i, j, k, PV_node,                         \
-                          GrGeomSolidData(grgeom),                  \
-                          GrGeomSolidOctreeBGLevel(grgeom) + r,     \
-                          ix, iy, iz, nx, ny, nz, sx, sy, sz,       \
-                          (GrGeomOctreeNodeIsOutside(PV_node) ||    \
-                           GrGeomOctreeNodeIsEmpty(PV_node)),       \
-                          body);                                    \
-  }
+#define GrGeomOutLoop2(i, j, k, grgeom,                                   \
+                       r, ix, iy, iz, nx, ny, nz, sx, sy, sz, body)       \
+        {                                                                 \
+          GrGeomOctree  *PV_node;                                         \
+          double PV_ref = pow(2.0, r);                                    \
+                                                                          \
+                                                                          \
+          i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;                  \
+          j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;                  \
+          k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;                  \
+          GrGeomOctreeNodeLoop2(i, j, k, PV_node,                         \
+                                GrGeomSolidData(grgeom),                  \
+                                GrGeomSolidOctreeBGLevel(grgeom) + r,     \
+                                ix, iy, iz, nx, ny, nz, sx, sy, sz,       \
+                                (GrGeomOctreeNodeIsOutside(PV_node) ||    \
+                                 GrGeomOctreeNodeIsEmpty(PV_node)),       \
+                                body);                                    \
+        }
 
 /*--------------------------------------------------------------------------
  * GrGeomSolid looping macro:
  *   Macro for looping over the faces of a solid surface.
  *--------------------------------------------------------------------------*/
 #if 0
-#define GrGeomSurfLoop(i, j, k, fdir, grgeom,                  \
-                       r, ix, iy, iz, nx, ny, nz, body)        \
-  {                                                            \
-    GrGeomOctree  *PV_node;                                    \
-    double PV_ref = pow(2.0, r);                               \
-                                                               \
-    i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;             \
-    j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;             \
-    k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;             \
-    GrGeomOctreeFaceLoop(i, j, k, fdir, PV_node,               \
-                         GrGeomSolidData(grgeom),              \
-                         GrGeomSolidOctreeBGLevel(grgeom) + r, \
-                         ix, iy, iz, nx, ny, nz, body);        \
-  }
+#define GrGeomSurfLoop(i, j, k, fdir, grgeom,                        \
+                       r, ix, iy, iz, nx, ny, nz, body)              \
+        {                                                            \
+          GrGeomOctree  *PV_node;                                    \
+          double PV_ref = pow(2.0, r);                               \
+                                                                     \
+          i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;             \
+          j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;             \
+          k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;             \
+          GrGeomOctreeFaceLoop(i, j, k, fdir, PV_node,               \
+                               GrGeomSolidData(grgeom),              \
+                               GrGeomSolidOctreeBGLevel(grgeom) + r, \
+                               ix, iy, iz, nx, ny, nz, body);        \
+        }
 
 #else
 
@@ -286,85 +286,85 @@ typedef struct {
 //
 
 #define GrGeomSurfLoopBoxes_default(i, j, k, fdir, grgeom, ix, iy, iz, nx, ny, nz, body) \
-  {                                                                              \
-    int PV_fdir[3];                                                              \
-                                                                                 \
-    fdir = PV_fdir;                                                              \
-    int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                          \
-    int *PV_visiting = NULL;                                                     \
-    PF_UNUSED(PV_visiting);                                                      \
-    for (int PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                      \
-    {                                                                            \
-      switch (PV_f)                                                              \
-      {                                                                          \
-        case GrGeomOctreeFaceL:                                                  \
-          fdir[0] = -1; fdir[1] = 0; fdir[2] = 0;                                \
-          break;                                                                 \
-        case GrGeomOctreeFaceR:                                                  \
-          fdir[0] = 1; fdir[1] = 0; fdir[2] = 0;                                 \
-          break;                                                                 \
-        case GrGeomOctreeFaceD:                                                  \
-          fdir[0] = 0; fdir[1] = -1; fdir[2] = 0;                                \
-          break;                                                                 \
-        case GrGeomOctreeFaceU:                                                  \
-          fdir[0] = 0; fdir[1] = 1; fdir[2] = 0;                                 \
-          break;                                                                 \
-        case GrGeomOctreeFaceB:                                                  \
-          fdir[0] = 0; fdir[1] = 0; fdir[2] = -1;                                \
-          break;                                                                 \
-        case GrGeomOctreeFaceF:                                                  \
-          fdir[0] = 0; fdir[1] = 0; fdir[2] = 1;                                 \
-          break;                                                                 \
-        default:                                                                 \
-          fdir[0] = -9999; fdir[1] = -9999; fdir[2] = -99999;                    \
-          break;                                                                 \
-      }                                                                          \
-                                                                                 \
-      BoxArray* boxes = GrGeomSolidSurfaceBoxes(grgeom, PV_f);                   \
-      for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)               \
-      {                                                                          \
-        Box box = BoxArrayGetBox(boxes, PV_box);                                 \
-        /* find octree and region intersection */                                \
-        PV_ixl = pfmax(ix, box.lo[0]);                                           \
-        PV_iyl = pfmax(iy, box.lo[1]);                                           \
-        PV_izl = pfmax(iz, box.lo[2]);                                           \
-        PV_ixu = pfmin((ix + nx - 1), box.up[0]);                                \
-        PV_iyu = pfmin((iy + ny - 1), box.up[1]);                                \
-        PV_izu = pfmin((iz + nz - 1), box.up[2]);                                \
-                                                                                 \
-        for (k = PV_izl; k <= PV_izu; k++)                                       \
-          for (j = PV_iyl; j <= PV_iyu; j++)                                     \
-            for (i = PV_ixl; i <= PV_ixu; i++)                                   \
-            {                                                                    \
-              body;                                                              \
-            }                                                                    \
-      }                                                                          \
-    }                                                                            \
-  }
-
-
-#define GrGeomSurfLoop(i, j, k, fdir, grgeom,                                \
-                       r, ix, iy, iz, nx, ny, nz, body)                      \
-  {                                                                          \
-    if (r == 0 && GrGeomSolidSurfaceBoxes(grgeom, GrGeomOctreeNumFaces - 1)) \
-    {                                                                        \
-      GrGeomSurfLoopBoxes(i, j, k, fdir, grgeom,                             \
-                          ix, iy, iz, nx, ny, nz, body);                     \
-    }                                                                        \
-    else                                                                     \
-    {                                                                        \
-      GrGeomOctree  *PV_node;                                                \
-      double PV_ref = pow(2.0, r);                                           \
-                                                                             \
-      i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;                         \
-      j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;                         \
-      k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;                         \
-      GrGeomOctreeFaceLoop(i, j, k, fdir, PV_node,                           \
-                           GrGeomSolidData(grgeom),                          \
-                           GrGeomSolidOctreeBGLevel(grgeom) + r,             \
-                           ix, iy, iz, nx, ny, nz, body);                    \
-    }                                                                        \
-  }
+        {                                                                                \
+          int PV_fdir[3];                                                                \
+                                                                                         \
+          fdir = PV_fdir;                                                                \
+          int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                            \
+          int *PV_visiting = NULL;                                                       \
+          PF_UNUSED(PV_visiting);                                                        \
+          for (int PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                        \
+          {                                                                              \
+            switch (PV_f)                                                                \
+            {                                                                            \
+              case GrGeomOctreeFaceL:                                                    \
+                fdir[0] = -1; fdir[1] = 0; fdir[2] = 0;                                  \
+                break;                                                                   \
+              case GrGeomOctreeFaceR:                                                    \
+                fdir[0] = 1; fdir[1] = 0; fdir[2] = 0;                                   \
+                break;                                                                   \
+              case GrGeomOctreeFaceD:                                                    \
+                fdir[0] = 0; fdir[1] = -1; fdir[2] = 0;                                  \
+                break;                                                                   \
+              case GrGeomOctreeFaceU:                                                    \
+                fdir[0] = 0; fdir[1] = 1; fdir[2] = 0;                                   \
+                break;                                                                   \
+              case GrGeomOctreeFaceB:                                                    \
+                fdir[0] = 0; fdir[1] = 0; fdir[2] = -1;                                  \
+                break;                                                                   \
+              case GrGeomOctreeFaceF:                                                    \
+                fdir[0] = 0; fdir[1] = 0; fdir[2] = 1;                                   \
+                break;                                                                   \
+              default:                                                                   \
+                fdir[0] = -9999; fdir[1] = -9999; fdir[2] = -99999;                      \
+                break;                                                                   \
+            }                                                                            \
+                                                                                         \
+            BoxArray* boxes = GrGeomSolidSurfaceBoxes(grgeom, PV_f);                     \
+            for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                 \
+            {                                                                            \
+              Box box = BoxArrayGetBox(boxes, PV_box);                                   \
+              /* find octree and region intersection */                                  \
+              PV_ixl = pfmax(ix, box.lo[0]);                                             \
+              PV_iyl = pfmax(iy, box.lo[1]);                                             \
+              PV_izl = pfmax(iz, box.lo[2]);                                             \
+              PV_ixu = pfmin((ix + nx - 1), box.up[0]);                                  \
+              PV_iyu = pfmin((iy + ny - 1), box.up[1]);                                  \
+              PV_izu = pfmin((iz + nz - 1), box.up[2]);                                  \
+                                                                                         \
+              for (k = PV_izl; k <= PV_izu; k++)                                         \
+              for (j = PV_iyl; j <= PV_iyu; j++)                                         \
+              for (i = PV_ixl; i <= PV_ixu; i++)                                         \
+              {                                                                          \
+                body;                                                                    \
+              }                                                                          \
+            }                                                                            \
+          }                                                                              \
+        }
+
+
+#define GrGeomSurfLoop(i, j, k, fdir, grgeom,                                      \
+                       r, ix, iy, iz, nx, ny, nz, body)                            \
+        {                                                                          \
+          if (r == 0 && GrGeomSolidSurfaceBoxes(grgeom, GrGeomOctreeNumFaces - 1)) \
+          {                                                                        \
+            GrGeomSurfLoopBoxes(i, j, k, fdir, grgeom,                             \
+                                ix, iy, iz, nx, ny, nz, body);                     \
+          }                                                                        \
+          else                                                                     \
+          {                                                                        \
+            GrGeomOctree  *PV_node;                                                \
+            double PV_ref = pow(2.0, r);                                           \
+                                                                                   \
+            i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;                         \
+            j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;                         \
+            k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;                         \
+            GrGeomOctreeFaceLoop(i, j, k, fdir, PV_node,                           \
+                                 GrGeomSolidData(grgeom),                          \
+                                 GrGeomSolidOctreeBGLevel(grgeom) + r,             \
+                                 ix, iy, iz, nx, ny, nz, body);                    \
+          }                                                                        \
+        }
 
 #endif
 
@@ -378,189 +378,189 @@ typedef struct {
 #if 1
 
 #define GrGeomPatchLoopBoxes_default(i, j, k, fdir, grgeom, patch_num, ix, iy, iz, nx, ny, nz, body) \
-  {                                                                                          \
-    int PV_fdir[3];                                                                          \
-                                                                                             \
-    fdir = PV_fdir;                                                                          \
-    int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                                      \
-    int *PV_visiting = NULL;                                                                 \
-    PF_UNUSED(PV_visiting);						                     \
-    for (int PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                                  \
-    {                                                                                        \
-      switch (PV_f)                                                                          \
-      {                                                                                      \
-        case GrGeomOctreeFaceL:                                                              \
-          fdir[0] = -1; fdir[1] = 0; fdir[2] = 0;                                            \
-          break;                                                                             \
-        case GrGeomOctreeFaceR:                                                              \
-          fdir[0] = 1; fdir[1] = 0; fdir[2] = 0;                                             \
-          break;                                                                             \
-        case GrGeomOctreeFaceD:                                                              \
-          fdir[0] = 0; fdir[1] = -1; fdir[2] = 0;                                            \
-          break;                                                                             \
-        case GrGeomOctreeFaceU:                                                              \
-          fdir[0] = 0; fdir[1] = 1; fdir[2] = 0;                                             \
-          break;                                                                             \
-        case GrGeomOctreeFaceB:                                                              \
-          fdir[0] = 0; fdir[1] = 0; fdir[2] = -1;                                            \
-          break;                                                                             \
-        case GrGeomOctreeFaceF:                                                              \
-          fdir[0] = 0; fdir[1] = 0; fdir[2] = 1;                                             \
-          break;                                                                             \
-        default:                                                                             \
-          fdir[0] = -9999; fdir[1] = -9999; fdir[2] = -99999;                                \
-          break;                                                                             \
-      }                                                                                      \
-                                                                                             \
-      BoxArray* boxes = GrGeomSolidPatchBoxes(grgeom, patch_num, PV_f);                      \
-      for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                           \
-      {                                                                                      \
-        Box box = BoxArrayGetBox(boxes, PV_box);                                             \
-        /* find octree and region intersection */                                            \
-        PV_ixl = pfmax(ix, box.lo[0]);                                                       \
-        PV_iyl = pfmax(iy, box.lo[1]);                                                       \
-        PV_izl = pfmax(iz, box.lo[2]);                                                       \
-        PV_ixu = pfmin((ix + nx - 1), box.up[0]);                                            \
-        PV_iyu = pfmin((iy + ny - 1), box.up[1]);                                            \
-        PV_izu = pfmin((iz + nz - 1), box.up[2]);                                            \
-                                                                                             \
-        for (k = PV_izl; k <= PV_izu; k++)                                                   \
-          for (j = PV_iyl; j <= PV_iyu; j++)                                                 \
-            for (i = PV_ixl; i <= PV_ixu; i++)                                               \
-            {                                                                                \
-              body;                                                                          \
-            }                                                                                \
-      }                                                                                      \
-    }                                                                                        \
-  }
-
-#define GrGeomPatchLoopBoxesNoFdir_default(i, j, k, grgeom, patch_num, ovrlnd,				\
-                                   ix, iy, iz, nx, ny, nz,              \
-                                   locals, setup,                       \
-                                   f_left, f_right,                     \
-                                   f_down, f_up,                        \
-                                   f_back, f_front,                     \
-                                   finalize)                            \
-  {                                                                     \
-    int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                 \
-    int *PV_visiting = NULL;                                            \
-    PF_UNUSED(PV_visiting);                                             \
-    UNPACK(locals);                                                     \
-                                                                        \
-    for (int PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)             \
-    {                                                                   \
-      BoxArray* boxes = GrGeomSolidPatchBoxes(grgeom, patch_num, PV_f); \
-      for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)      \
-      {                                                                 \
-        Box box = BoxArrayGetBox(boxes, PV_box);                        \
-        /* find octree and region intersection */                       \
-        PV_ixl = pfmax(ix, box.lo[0]);                                  \
-        PV_iyl = pfmax(iy, box.lo[1]);                                  \
-        PV_izl = pfmax(iz, box.lo[2]);                                  \
-        PV_ixu = pfmin((ix + nx - 1), box.up[0]);                       \
-        PV_iyu = pfmin((iy + ny - 1), box.up[1]);                       \
-        PV_izu = pfmin((iz + nz - 1), box.up[2]);                       \
-                                                                        \
-        for (k = PV_izl; k <= PV_izu; k++)                              \
-          for (j = PV_iyl; j <= PV_iyu; j++)                            \
-            for (i = PV_ixl; i <= PV_ixu; i++)                          \
-            {                                                           \
-              setup;                                                    \
-              switch(PV_f)                                              \
-              {                                                         \
-                f_left;                                                 \
-                f_right;                                                \
-                f_down;                                                 \
-                f_up;                                                   \
-                f_back;                                                 \
-                f_front;                                                \
-              }                                                         \
-              finalize;                                                 \
-            }                                                           \
-      }                                                                 \
-    }                                                                   \
-  }
-
-#define GrGeomPatchLoop(i, j, k, fdir, grgeom, patch_num,                             \
-                        r, ix, iy, iz, nx, ny, nz, body)                              \
-  {                                                                                   \
-    if (r == 0 && GrGeomSolidPatchBoxes(grgeom, patch_num, GrGeomOctreeNumFaces - 1)) \
-    {                                                                                 \
-      GrGeomPatchLoopBoxes(i, j, k, fdir, grgeom, patch_num,                          \
-                           ix, iy, iz, nx, ny, nz, body);                             \
-    }                                                                                 \
-    else                                                                              \
-    {                                                                                 \
-      GrGeomOctree  *PV_node;                                                         \
-      double PV_ref = pow(2.0, r);                                                    \
-                                                                                      \
-                                                                                      \
-      i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;                                  \
-      j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;                                  \
-      k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;                                  \
-      GrGeomOctreeFaceLoop(i, j, k, fdir, PV_node,                                    \
-                           GrGeomSolidPatch(grgeom, patch_num),                       \
-                           GrGeomSolidOctreeBGLevel(grgeom) + r,                      \
-                           ix, iy, iz, nx, ny, nz, body);                             \
-    }                                                                                 \
-  }
-
-#define GrGeomPatchLoopNoFdir(i, j, k, grgeom, patch_num, ovrlnd,       \
-                              r, ix, iy, iz, nx, ny, nz,                \
-                              locals, setup,                            \
-                              f_left, f_right,                          \
-                              f_down, f_up,                             \
-                              f_back, f_front,                          \
-                              finalize)                                 \
-  {                                                                     \
-    if (r == 0 && GrGeomSolidPatchBoxes(grgeom, patch_num, GrGeomOctreeNumFaces - 1)) \
-    {                                                                   \
-      GrGeomPatchLoopBoxesNoFdir(i, j, k, grgeom, patch_num, ovrlnd,    \
-                                 ix, iy, iz, nx, ny, nz,                \
-                                 locals, setup,                         \
-                                 f_left, f_right,                       \
-                                 f_down, f_up,                          \
-                                 f_back, f_front,                       \
-                                 finalize)                              \
-    }                                                                   \
-    else                                                                \
-    {                                                                   \
-      GrGeomOctree  *PV_node;                                           \
-      double PV_ref = pow(2.0, r);                                      \
-                                                                        \
-                                                                        \
-      i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;                    \
-      j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;                    \
-      k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;                    \
-      GrGeomOctreeFaceLoopNoFdir(i, j, k, PV_node,                      \
-                                 GrGeomSolidPatch(grgeom, patch_num),   \
-                                 GrGeomSolidOctreeBGLevel(grgeom) + r,  \
-                                 ix, iy, iz, nx, ny, nz,                \
-                                 locals, setup,                         \
-                                 f_left, f_right,                       \
-                                 f_down, f_up,                          \
-                                 f_back, f_front,                       \
-                                 finalize)                              \
-    }                                                                   \
-  }
+        {                                                                                            \
+          int PV_fdir[3];                                                                            \
+                                                                                                     \
+          fdir = PV_fdir;                                                                            \
+          int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                                        \
+          int *PV_visiting = NULL;                                                                   \
+          PF_UNUSED(PV_visiting);                                                                    \
+          for (int PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                                    \
+          {                                                                                          \
+            switch (PV_f)                                                                            \
+            {                                                                                        \
+              case GrGeomOctreeFaceL:                                                                \
+                fdir[0] = -1; fdir[1] = 0; fdir[2] = 0;                                              \
+                break;                                                                               \
+              case GrGeomOctreeFaceR:                                                                \
+                fdir[0] = 1; fdir[1] = 0; fdir[2] = 0;                                               \
+                break;                                                                               \
+              case GrGeomOctreeFaceD:                                                                \
+                fdir[0] = 0; fdir[1] = -1; fdir[2] = 0;                                              \
+                break;                                                                               \
+              case GrGeomOctreeFaceU:                                                                \
+                fdir[0] = 0; fdir[1] = 1; fdir[2] = 0;                                               \
+                break;                                                                               \
+              case GrGeomOctreeFaceB:                                                                \
+                fdir[0] = 0; fdir[1] = 0; fdir[2] = -1;                                              \
+                break;                                                                               \
+              case GrGeomOctreeFaceF:                                                                \
+                fdir[0] = 0; fdir[1] = 0; fdir[2] = 1;                                               \
+                break;                                                                               \
+              default:                                                                               \
+                fdir[0] = -9999; fdir[1] = -9999; fdir[2] = -99999;                                  \
+                break;                                                                               \
+            }                                                                                        \
+                                                                                                     \
+            BoxArray* boxes = GrGeomSolidPatchBoxes(grgeom, patch_num, PV_f);                        \
+            for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                             \
+            {                                                                                        \
+              Box box = BoxArrayGetBox(boxes, PV_box);                                               \
+              /* find octree and region intersection */                                              \
+              PV_ixl = pfmax(ix, box.lo[0]);                                                         \
+              PV_iyl = pfmax(iy, box.lo[1]);                                                         \
+              PV_izl = pfmax(iz, box.lo[2]);                                                         \
+              PV_ixu = pfmin((ix + nx - 1), box.up[0]);                                              \
+              PV_iyu = pfmin((iy + ny - 1), box.up[1]);                                              \
+              PV_izu = pfmin((iz + nz - 1), box.up[2]);                                              \
+                                                                                                     \
+              for (k = PV_izl; k <= PV_izu; k++)                                                     \
+              for (j = PV_iyl; j <= PV_iyu; j++)                                                     \
+              for (i = PV_ixl; i <= PV_ixu; i++)                                                     \
+              {                                                                                      \
+                body;                                                                                \
+              }                                                                                      \
+            }                                                                                        \
+          }                                                                                          \
+        }
+
+#define GrGeomPatchLoopBoxesNoFdir_default(i, j, k, grgeom, patch_num, ovrlnd,                          \
+                                           ix, iy, iz, nx, ny, nz,                                      \
+                                           locals, setup,                                               \
+                                           f_left, f_right,                                             \
+                                           f_down, f_up,                                                \
+                                           f_back, f_front,                                             \
+                                           finalize)                                                    \
+        {                                                                                               \
+          int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                                           \
+          int *PV_visiting = NULL;                                                                      \
+          PF_UNUSED(PV_visiting);                                                                       \
+          UNPACK(locals);                                                                               \
+                                                                                                        \
+          for (int PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                                       \
+          {                                                                                             \
+            BoxArray* boxes = GrGeomSolidPatchBoxes(grgeom, patch_num, PV_f);                           \
+            for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                                \
+            {                                                                                           \
+              Box box = BoxArrayGetBox(boxes, PV_box);                                                  \
+              /* find octree and region intersection */                                                 \
+              PV_ixl = pfmax(ix, box.lo[0]);                                                            \
+              PV_iyl = pfmax(iy, box.lo[1]);                                                            \
+              PV_izl = pfmax(iz, box.lo[2]);                                                            \
+              PV_ixu = pfmin((ix + nx - 1), box.up[0]);                                                 \
+              PV_iyu = pfmin((iy + ny - 1), box.up[1]);                                                 \
+              PV_izu = pfmin((iz + nz - 1), box.up[2]);                                                 \
+                                                                                                        \
+              for (k = PV_izl; k <= PV_izu; k++)                                                        \
+              for (j = PV_iyl; j <= PV_iyu; j++)                                                        \
+              for (i = PV_ixl; i <= PV_ixu; i++)                                                        \
+              {                                                                                         \
+                setup;                                                                                  \
+                switch (PV_f)                                                                           \
+                {                                                                                       \
+                f_left;                                                                                 \
+                f_right;                                                                                \
+                f_down;                                                                                 \
+                f_up;                                                                                   \
+                f_back;                                                                                 \
+                f_front;                                                                                \
+                }                                                                                       \
+                finalize;                                                                               \
+              }                                                                                         \
+            }                                                                                           \
+          }                                                                                             \
+        }
+
+#define GrGeomPatchLoop(i, j, k, fdir, grgeom, patch_num,                                   \
+                        r, ix, iy, iz, nx, ny, nz, body)                                    \
+        {                                                                                   \
+          if (r == 0 && GrGeomSolidPatchBoxes(grgeom, patch_num, GrGeomOctreeNumFaces - 1)) \
+          {                                                                                 \
+            GrGeomPatchLoopBoxes(i, j, k, fdir, grgeom, patch_num,                          \
+                                 ix, iy, iz, nx, ny, nz, body);                             \
+          }                                                                                 \
+          else                                                                              \
+          {                                                                                 \
+            GrGeomOctree  *PV_node;                                                         \
+            double PV_ref = pow(2.0, r);                                                    \
+                                                                                            \
+                                                                                            \
+            i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;                                  \
+            j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;                                  \
+            k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;                                  \
+            GrGeomOctreeFaceLoop(i, j, k, fdir, PV_node,                                    \
+                                 GrGeomSolidPatch(grgeom, patch_num),                       \
+                                 GrGeomSolidOctreeBGLevel(grgeom) + r,                      \
+                                 ix, iy, iz, nx, ny, nz, body);                             \
+          }                                                                                 \
+        }
+
+#define GrGeomPatchLoopNoFdir(i, j, k, grgeom, patch_num, ovrlnd,                           \
+                              r, ix, iy, iz, nx, ny, nz,                                    \
+                              locals, setup,                                                \
+                              f_left, f_right,                                              \
+                              f_down, f_up,                                                 \
+                              f_back, f_front,                                              \
+                              finalize)                                                     \
+        {                                                                                   \
+          if (r == 0 && GrGeomSolidPatchBoxes(grgeom, patch_num, GrGeomOctreeNumFaces - 1)) \
+          {                                                                                 \
+            GrGeomPatchLoopBoxesNoFdir(i, j, k, grgeom, patch_num, ovrlnd,                  \
+                                       ix, iy, iz, nx, ny, nz,                              \
+                                       locals, setup,                                       \
+                                       f_left, f_right,                                     \
+                                       f_down, f_up,                                        \
+                                       f_back, f_front,                                     \
+                                       finalize)                                            \
+          }                                                                                 \
+          else                                                                              \
+          {                                                                                 \
+            GrGeomOctree  *PV_node;                                                         \
+            double PV_ref = pow(2.0, r);                                                    \
+                                                                                            \
+                                                                                            \
+            i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;                                  \
+            j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;                                  \
+            k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;                                  \
+            GrGeomOctreeFaceLoopNoFdir(i, j, k, PV_node,                                    \
+                                       GrGeomSolidPatch(grgeom, patch_num),                 \
+                                       GrGeomSolidOctreeBGLevel(grgeom) + r,                \
+                                       ix, iy, iz, nx, ny, nz,                              \
+                                       locals, setup,                                       \
+                                       f_left, f_right,                                     \
+                                       f_down, f_up,                                        \
+                                       f_back, f_front,                                     \
+                                       finalize)                                            \
+          }                                                                                 \
+        }
 
 #else
 
-#define GrGeomPatchLoop(i, j, k, fdir, grgeom, patch_num,      \
-                        r, ix, iy, iz, nx, ny, nz, body)       \
-  {                                                            \
-    GrGeomOctree  *PV_node;                                    \
-    double PV_ref = pow(2.0, r);                               \
-                                                               \
-                                                               \
-    i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;             \
-    j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;             \
-    k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;             \
-    GrGeomOctreeFaceLoop(i, j, k, fdir, PV_node,               \
-                         GrGeomSolidPatch(grgeom, patch_num),  \
-                         GrGeomSolidOctreeBGLevel(grgeom) + r, \
-                         ix, iy, iz, nx, ny, nz, body);        \
-  }
+#define GrGeomPatchLoop(i, j, k, fdir, grgeom, patch_num,            \
+                        r, ix, iy, iz, nx, ny, nz, body)             \
+        {                                                            \
+          GrGeomOctree  *PV_node;                                    \
+          double PV_ref = pow(2.0, r);                               \
+                                                                     \
+                                                                     \
+          i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;             \
+          j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;             \
+          k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;             \
+          GrGeomOctreeFaceLoop(i, j, k, fdir, PV_node,               \
+                               GrGeomSolidPatch(grgeom, patch_num),  \
+                               GrGeomSolidOctreeBGLevel(grgeom) + r, \
+                               ix, iy, iz, nx, ny, nz, body);        \
+        }
 
 #endif
 
@@ -593,37 +593,37 @@ typedef struct {
 // cells are being looped over which would be really bad for
 // performance reasons.
 
-#define GrGeomInBoxLoop(                                          \
-                        i, j, k,                                  \
-                        num_i, num_j, num_k,                      \
-                        grgeom, box_size_power,                   \
-                        ix, iy, iz, nx, ny, nz,                   \
-                        body)                                     \
-  {                                                               \
-    GrGeomOctree  *PV_node;                                       \
-    int PV_level_of_interest;                                     \
-    PV_level_of_interest = GrGeomSolidOctreeBGLevel(grgeom) -     \
-                           box_size_power - 1;                    \
-    PV_level_of_interest = pfmax(0, PV_level_of_interest);        \
-                                                                  \
-    i = GrGeomSolidOctreeIX(grgeom);                              \
-    j = GrGeomSolidOctreeIY(grgeom);                              \
-    k = GrGeomSolidOctreeIZ(grgeom);                              \
-                                                                  \
-    GrGeomOctreeNodeBoxLoop(i, j, k,                              \
-                            num_i, num_j, num_k,                  \
-                            PV_node,                              \
-                            GrGeomSolidData(grgeom),              \
-                            GrGeomSolidOctreeBGLevel(grgeom),     \
-                            PV_level_of_interest,                 \
-                            ix, iy, iz, nx, ny, nz,               \
-                            (GrGeomOctreeHasChildren(PV_node) ||  \
-                             GrGeomOctreeNodeIsInside(PV_node) || \
-                             GrGeomOctreeNodeIsFull(PV_node)),    \
-    {                                                             \
-      body;                                                       \
-    });                                                           \
-  }
+#define GrGeomInBoxLoop(                                                \
+                        i, j, k,                                        \
+                        num_i, num_j, num_k,                            \
+                        grgeom, box_size_power,                         \
+                        ix, iy, iz, nx, ny, nz,                         \
+                        body)                                           \
+        {                                                               \
+          GrGeomOctree  *PV_node;                                       \
+          int PV_level_of_interest;                                     \
+          PV_level_of_interest = GrGeomSolidOctreeBGLevel(grgeom) -     \
+                                 box_size_power - 1;                    \
+          PV_level_of_interest = pfmax(0, PV_level_of_interest);        \
+                                                                        \
+          i = GrGeomSolidOctreeIX(grgeom);                              \
+          j = GrGeomSolidOctreeIY(grgeom);                              \
+          k = GrGeomSolidOctreeIZ(grgeom);                              \
+                                                                        \
+          GrGeomOctreeNodeBoxLoop(i, j, k,                              \
+                                  num_i, num_j, num_k,                  \
+                                  PV_node,                              \
+                                  GrGeomSolidData(grgeom),              \
+                                  GrGeomSolidOctreeBGLevel(grgeom),     \
+                                  PV_level_of_interest,                 \
+                                  ix, iy, iz, nx, ny, nz,               \
+                                  (GrGeomOctreeHasChildren(PV_node) ||  \
+                                   GrGeomOctreeNodeIsInside(PV_node) || \
+                                   GrGeomOctreeNodeIsFull(PV_node)),    \
+    {                                                                   \
+      body;                                                             \
+    });                                                                 \
+        }
 
 
 
diff --git a/pfsimulator/parflow_lib/grid.c b/pfsimulator/parflow_lib/grid.c
index 0501ceec5..2e11b350b 100644
--- a/pfsimulator/parflow_lib/grid.c
+++ b/pfsimulator/parflow_lib/grid.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Member functions for Grid class.
diff --git a/pfsimulator/parflow_lib/grid.h b/pfsimulator/parflow_lib/grid.h
index 9e11bd679..05a3e0fd2 100644
--- a/pfsimulator/parflow_lib/grid.h
+++ b/pfsimulator/parflow_lib/grid.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Header info for the Grid structures
@@ -103,7 +103,7 @@ typedef struct {
  *--------------------------------------------------------------------------*/
 
 #define SubgridArraySubgrid(subgrid_array, i) \
-  ((Subgrid*)SubregionArraySubregion(subgrid_array, i))
+        ((Subgrid*)SubregionArraySubregion(subgrid_array, i))
 #define SubgridArraySize(subgrid_array)  SubregionArraySize(subgrid_array)
 
 /*--------------------------------------------------------------------------
@@ -128,18 +128,18 @@ typedef struct {
  *--------------------------------------------------------------------------*/
 
 #define SubgridDX(subgrid) \
-  RealSpaceDX(SubgridRX(subgrid))
+        RealSpaceDX(SubgridRX(subgrid))
 #define SubgridDY(subgrid) \
-  RealSpaceDY(SubgridRY(subgrid))
+        RealSpaceDY(SubgridRY(subgrid))
 #define SubgridDZ(subgrid) \
-  RealSpaceDZ(SubgridRZ(subgrid))
+        RealSpaceDZ(SubgridRZ(subgrid))
 
 #define SubgridX(subgrid) \
-  RealSpaceX(SubgridIX(subgrid), SubgridRX(subgrid))
+        RealSpaceX(SubgridIX(subgrid), SubgridRX(subgrid))
 #define SubgridY(subgrid) \
-  RealSpaceY(SubgridIY(subgrid), SubgridRY(subgrid))
+        RealSpaceY(SubgridIY(subgrid), SubgridRY(subgrid))
 #define SubgridZ(subgrid) \
-  RealSpaceZ(SubgridIZ(subgrid), SubgridRZ(subgrid))
+        RealSpaceZ(SubgridIZ(subgrid), SubgridRZ(subgrid))
 
 /*--------------------------------------------------------------------------
  * Looping macros:
@@ -152,32 +152,32 @@ typedef struct {
  *--------------------------------------------------------------------------*/
 
 #define NewSubgrid(x, y, z, nx, ny, nz, rx, ry, rz, process) \
-  ((Subgrid*)NewSubregion(x, y, z, nx, ny, nz, 1, 1, 1, rx, ry, rz, process))
+        ((Subgrid*)NewSubregion(x, y, z, nx, ny, nz, 1, 1, 1, rx, ry, rz, process))
 
 #define NewSubgridArray()  ((SubgridArray*)NewSubregionArray())
 
 #define FreeSubgrid(subgrid)  FreeSubregion((Subregion*)subgrid)
 
 #define FreeSubgridArray(subgrid_array) \
-  FreeSubregionArray((SubregionArray*)subgrid_array)
+        FreeSubregionArray((SubregionArray*)subgrid_array)
 
 #define DuplicateSubgrid(subgrid) \
-  ((Subgrid*)DuplicateSubregion((Subregion*)subgrid))
+        ((Subgrid*)DuplicateSubregion((Subregion*)subgrid))
 
 #define AppendSubgrid(subgrid, subgrid_array) \
-  AppendSubregion((Subregion*)subgrid, (SubregionArray*)subgrid_array)
+        AppendSubregion((Subregion*)subgrid, (SubregionArray*)subgrid_array)
 
-#define AppendSubgridArray(subgrid_array_0, subgrid_array_1) \
-  AppendSubregionArray((SubregionArray*)subgrid_array_0,     \
-                       (SubregionArray*)subgrid_array_1)
+#define AppendSubgridArray(subgrid_array_0, subgrid_array_1)       \
+        AppendSubregionArray((SubregionArray*)subgrid_array_0,     \
+                             (SubregionArray*)subgrid_array_1)
 
 #define ConvertToSubregion(subgrid)  ((Subregion*)subgrid)
 
-#define SubgridEltIndex(subgrid, x, y, z) \
-  (((x) - SubgridIX(subgrid)) +           \
-   (((y) - SubgridIY(subgrid)) +          \
-    (((z) - SubgridIZ(subgrid))) *        \
-    SubgridNY(subgrid)) *                 \
-   SubgridNX(subgrid))
+#define SubgridEltIndex(subgrid, x, y, z)       \
+        (((x) - SubgridIX(subgrid)) +           \
+         (((y) - SubgridIY(subgrid)) +          \
+          (((z) - SubgridIZ(subgrid))) *        \
+          SubgridNY(subgrid)) *                 \
+         SubgridNX(subgrid))
 
 #endif
diff --git a/pfsimulator/parflow_lib/hbt.c b/pfsimulator/parflow_lib/hbt.c
index ecadf8658..9f3af0db7 100644
--- a/pfsimulator/parflow_lib/hbt.c
+++ b/pfsimulator/parflow_lib/hbt.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "hbt.h"
 
@@ -241,7 +241,7 @@ int HBT_insert(
                int   sizeof_obj)
 {
   HBT_element *temp, *inserted, *rebalance_son, *rebalance,
-    *rebalance_father;
+              *rebalance_father;
   int done = 0;
   int test, test_rebalance;
   short rebalance_B;
@@ -995,13 +995,13 @@ void HBT_scanf(
   void *obj;
   int size;
 
-  if(fscanf(file, "%d", &(height)) != 1)
+  if (fscanf(file, "%d", &(height)) != 1)
   {
     printf("ERROR: HBT_scanf failed to read height\n");
     abort();
   }
-  
-  if(fscanf(file, "%d", &(num)) != 1)
+
+  if (fscanf(file, "%d", &(num)) != 1)
   {
     printf("ERROR: HBT_scanf failed to read num\n");
     abort();
diff --git a/pfsimulator/parflow_lib/hbt.h b/pfsimulator/parflow_lib/hbt.h
index 8f6ac1e25..bc0bc097e 100644
--- a/pfsimulator/parflow_lib/hbt.h
+++ b/pfsimulator/parflow_lib/hbt.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #ifndef _hbt_HEADER
 #define _hbt_HEADER
 
diff --git a/pfsimulator/parflow_lib/hypre_dependences.h b/pfsimulator/parflow_lib/hypre_dependences.h
index 13f493d68..c798fd59b 100644
--- a/pfsimulator/parflow_lib/hypre_dependences.h
+++ b/pfsimulator/parflow_lib/hypre_dependences.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "HYPRE_struct_mv.h"
 #include "HYPRE_struct_ls.h"
diff --git a/pfsimulator/parflow_lib/index_space.c b/pfsimulator/parflow_lib/index_space.c
index ff3a50718..39091e9ac 100644
--- a/pfsimulator/parflow_lib/index_space.c
+++ b/pfsimulator/parflow_lib/index_space.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2019, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 #include "index_space.h"
@@ -100,9 +100,9 @@ BoxArray* NewBoxArray(BoxList *box_list)
       BoxCopy(&(box_array->boxes[i]), &(element->box));
       for (int dim = 0; dim < DIM; dim++)
       {
-        if(i == 0 || box_array->boxlimits[dim] > box_array->boxes[i].lo[dim])
+        if (i == 0 || box_array->boxlimits[dim] > box_array->boxes[i].lo[dim])
           box_array->boxlimits[dim] = box_array->boxes[i].lo[dim];
-        if(i == 0 || box_array->boxlimits[DIM + dim] < box_array->boxes[i].up[dim])
+        if (i == 0 || box_array->boxlimits[DIM + dim] < box_array->boxes[i].up[dim])
           box_array->boxlimits[DIM + dim] = box_array->boxes[i].up[dim];
       }
       element = element->next;
diff --git a/pfsimulator/parflow_lib/index_space.h b/pfsimulator/parflow_lib/index_space.h
index eb7207cb4..51eed1add 100644
--- a/pfsimulator/parflow_lib/index_space.h
+++ b/pfsimulator/parflow_lib/index_space.h
@@ -1,31 +1,31 @@
 
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2019, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef _INDEX_SPACE_HEADER
 #define _INDEX_SPACE_HEADER
diff --git a/pfsimulator/parflow_lib/infinity_norm.c b/pfsimulator/parflow_lib/infinity_norm.c
index d656d8489..18921b25d 100644
--- a/pfsimulator/parflow_lib/infinity_norm.c
+++ b/pfsimulator/parflow_lib/infinity_norm.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Infinity Norm of two vectors
 *
diff --git a/pfsimulator/parflow_lib/info_header.h b/pfsimulator/parflow_lib/info_header.h
index 9e20f2c9b..3a766d81a 100644
--- a/pfsimulator/parflow_lib/info_header.h
+++ b/pfsimulator/parflow_lib/info_header.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*
  * Steven F. Ashby
  *    Lawrence Livermore National Laboratory
diff --git a/pfsimulator/parflow_lib/innerprod.c b/pfsimulator/parflow_lib/innerprod.c
index 7717f1b6c..d8849e3cb 100644
--- a/pfsimulator/parflow_lib/innerprod.c
+++ b/pfsimulator/parflow_lib/innerprod.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Inner Product of two vectors
 *
@@ -53,6 +53,7 @@ double   InnerProd(
   int i_s, i, j, k, iv;
 
   amps_Invoice result_invoice;
+
   result_invoice = amps_NewInvoice("%d", &result);
 
   ForSubgridI(i_s, GridSubgrids(grid))
@@ -80,8 +81,8 @@ double   InnerProd(
     iv = 0;
 
     BoxLoopReduceI1(result,
-										i, j, k, ix, iy, iz, nx, ny, nz,
-										iv, nx_v, ny_v, nz_v, 1, 1, 1,
+                    i, j, k, ix, iy, iz, nx, ny, nz,
+                    iv, nx_v, ny_v, nz_v, 1, 1, 1,
     {
       ReduceSum(result, yp[iv] * xp[iv]);
     });
diff --git a/pfsimulator/parflow_lib/inputRF.c b/pfsimulator/parflow_lib/inputRF.c
index b8a4be3a6..ee48622f7 100644
--- a/pfsimulator/parflow_lib/inputRF.c
+++ b/pfsimulator/parflow_lib/inputRF.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
diff --git a/pfsimulator/parflow_lib/input_checks.c b/pfsimulator/parflow_lib/input_checks.c
index dd29715b6..dccf16b7f 100644
--- a/pfsimulator/parflow_lib/input_checks.c
+++ b/pfsimulator/parflow_lib/input_checks.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
diff --git a/pfsimulator/parflow_lib/input_database.c b/pfsimulator/parflow_lib/input_database.c
index 858f6a0b8..96cf09334 100644
--- a/pfsimulator/parflow_lib/input_database.c
+++ b/pfsimulator/parflow_lib/input_database.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -115,13 +115,14 @@ IDB *IDB_NewDB(char *filename)
   {
     /* Read the key and value from the input file */
     amps_SFBCast(amps_CommWorld, file, invoice);
-    if ((value_len + 1) > IDB_MAX_VALUE_LEN) {
+    if ((value_len + 1) > IDB_MAX_VALUE_LEN)
+    {
       key[key_len] = '\0';
       char s[128];
-      sprintf(s, "%d", IDB_MAX_VALUE_LEN-1);
+      sprintf(s, "%d", IDB_MAX_VALUE_LEN - 1);
       InputError("Error: The value associated with input database "
                  "key <%s> is too long. The maximum length is %s. ",
-                 key, s); 
+                 key, s);
     }
     key[key_len] = '\0';
     value[value_len] = '\0';
diff --git a/pfsimulator/parflow_lib/input_database.h b/pfsimulator/parflow_lib/input_database.h
index 8a645edd8..501a7799c 100644
--- a/pfsimulator/parflow_lib/input_database.h
+++ b/pfsimulator/parflow_lib/input_database.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef _INPUT_DATABASE_HEADER
 #define _INPUT_DATABASE_HEADER
@@ -74,8 +74,8 @@ typedef struct NameArray__ {
 typedef NameArrayStruct *NameArray;
 
 /**
- * Prints out a database entry.  
-
+ * Prints out a database entry.
+ *
  * If it was not used during a run it is flagged as being unused.
  *
  * @param file File pointer
@@ -219,7 +219,7 @@ int IDB_GetInt(IDB *database, const char *key);
 
 /**
  * Construct a name array from an input string.
-
+ *
  * Provided string is split by whitespace and entries are created for
  * each space separated words.
  *
@@ -260,7 +260,7 @@ int NA_NameToIndex(NameArray name_array, char *name);
 
 /**
  * Returns Index in the name array for the specified name, exit if not found
-
+ *
  * Prints error message and exits if name is not found.
  *
  * @param name_array  The name array for the lookup
@@ -288,9 +288,9 @@ void InputError(const char *format, const char *s1, const char *s2);
 
 /**
  * Output error for invalid value provided to a NameArray key and exits.
- * 
- * @param name_array the NameArray 
- * @param switch_name the invalid value provided 
+ *
+ * @param name_array the NameArray
+ * @param switch_name the invalid value provided
  * @param key the database key being parsed
  */
 void NA_InputError(NameArray name_array, const char *switch_name, const char *key);
diff --git a/pfsimulator/parflow_lib/input_porosity.c b/pfsimulator/parflow_lib/input_porosity.c
index 3390454f4..0bddfb73c 100644
--- a/pfsimulator/parflow_lib/input_porosity.c
+++ b/pfsimulator/parflow_lib/input_porosity.c
@@ -1,24 +1,30 @@
-/*BHEADER*********************************************************************
- * Copyright (c) 1995-2009, Lawrence Livermore National Security
- * LLC. Produced at the Lawrence Livermore National Laboratory. Written
- * by the Parflow Team (see the CONTRIBUTORS file)
- * <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- * This file is part of Parflow. For details, see
- * http://www.llnl.gov/casc/parflow
- * Please read the COPYRIGHT file or Our Notice and the LICENSE file
- * for the GNU Lesser General Public License.
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License (as published
- * by the Free Software Foundation) version 2.1 dated February 1999.
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- * and conditions of the GNU General Public License for more details.
- * You should have received a copy of the GNU Lesser General Public
- * License along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- * USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "parflow.h"
 /* helper type for reading from file */
 typedef struct {
diff --git a/pfsimulator/parflow_lib/kinsol_dependences.h b/pfsimulator/parflow_lib/kinsol_dependences.h
index c868bc978..649c84d92 100644
--- a/pfsimulator/parflow_lib/kinsol_dependences.h
+++ b/pfsimulator/parflow_lib/kinsol_dependences.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "../kinsol/kinsol.h"
 #include "../kinsol/iterativ.h"
diff --git a/pfsimulator/parflow_lib/kinsol_nonlin_solver.c b/pfsimulator/parflow_lib/kinsol_nonlin_solver.c
index 83b8033e8..5787ff126 100644
--- a/pfsimulator/parflow_lib/kinsol_nonlin_solver.c
+++ b/pfsimulator/parflow_lib/kinsol_nonlin_solver.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 #include "kinsol_dependences.h"
@@ -542,8 +542,8 @@ PFModule  *KinsolNonlinSolverNewPublicXtra()
   sprintf(key, "Solver.Nonlinear.PrintFlag");
   switch_name = GetStringDefault(key, "LowVerbosity");
   (public_xtra->print_flag) = NA_NameToIndexExitOnError(verbosity_switch_na,
-							switch_name,
-							key);
+                                                        switch_name,
+                                                        key);
   NA_FreeNameArray(verbosity_switch_na);
 
   eta_switch_na = NA_NewNameArray("EtaConstant Walker1 Walker2");
diff --git a/pfsimulator/parflow_lib/kinsol_pc.c b/pfsimulator/parflow_lib/kinsol_pc.c
index def622e2a..a28d97272 100644
--- a/pfsimulator/parflow_lib/kinsol_pc.c
+++ b/pfsimulator/parflow_lib/kinsol_pc.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 #include "kinsol_dependences.h"
@@ -302,6 +302,7 @@ PFModule  *KinsolPCNewPublicXtra(char *name, char *pc_name)
 #endif
       break;
     }
+
     default:
     {
       InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
diff --git a/pfsimulator/parflow_lib/l2_error_norm.c b/pfsimulator/parflow_lib/l2_error_norm.c
index 40acbfa02..12bdd9abd 100644
--- a/pfsimulator/parflow_lib/l2_error_norm.c
+++ b/pfsimulator/parflow_lib/l2_error_norm.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
diff --git a/pfsimulator/parflow_lib/line_process.c b/pfsimulator/parflow_lib/line_process.c
index 771d43b56..608243c8b 100644
--- a/pfsimulator/parflow_lib/line_process.c
+++ b/pfsimulator/parflow_lib/line_process.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routines for generating line processes for the turning bands method.
diff --git a/pfsimulator/parflow_lib/logging.c b/pfsimulator/parflow_lib/logging.c
index 7e0e2874a..fe24b0e82 100644
--- a/pfsimulator/parflow_lib/logging.c
+++ b/pfsimulator/parflow_lib/logging.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * NewLogging, FreeLogging, OpenLogFile, CloseLogFile
 *
@@ -132,7 +132,7 @@ void PrintVersionInfo(FILE *log_file)
 {
   fprintf(log_file, "\tVersion         : %s\n", PARFLOW_VERSION_STRING);
   fprintf(log_file, "\tCompiled on     : %s %s\n", __DATE__, __TIME__);
-  
+
 #ifdef CFLAGS
   fprintf(log_file, "\tWith C flags    : %s\n", CFLAGS);
 #endif
@@ -152,5 +152,4 @@ void PrintVersionInfo(FILE *log_file)
 #elif defined(PARFLOW_HAVE_OMP)
   fprintf(log_file, "\tWith acc backend: OMP\n");
 #endif
-
 }
diff --git a/pfsimulator/parflow_lib/logging.h b/pfsimulator/parflow_lib/logging.h
index eaf04cb6b..78899c6b4 100644
--- a/pfsimulator/parflow_lib/logging.h
+++ b/pfsimulator/parflow_lib/logging.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Header info for logging
@@ -35,6 +35,6 @@
 #define _LOGGING_HEADER
 
 #define IfLogging(level) \
-  if ((level <= GlobalsLoggingLevel) && (!amps_Rank(amps_CommWorld)))
+        if ((level <= GlobalsLoggingLevel) && (!amps_Rank(amps_CommWorld)))
 
 #endif
diff --git a/pfsimulator/parflow_lib/loops.h b/pfsimulator/parflow_lib/loops.h
index 2910ea19e..68c3ba4f4 100644
--- a/pfsimulator/parflow_lib/loops.h
+++ b/pfsimulator/parflow_lib/loops.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /** @file
  * @brief Generic loop macros
@@ -53,11 +53,11 @@
  * @param[in] sy Y striding factor
  * @param[in] sz Z striding factor
  */
-#define DeclareInc(jinc, kinc, nx, ny, nz, nxd, nyd, nzd, sx, sy, sz) \
-  int jinc = (sy) * (nxd) - (nx) * (sx);                              \
-  int kinc = (sz) * (nxd) * (nyd) - (ny) * (sy) * (nxd);              \
-  PF_UNUSED(nz);                                                      \
-  PF_UNUSED(nzd)
+#define DeclareInc(jinc, kinc, nx, ny, nz, nxd, nyd, nzd, sx, sy, sz)       \
+        int jinc = (sy) * (nxd) - (nx) * (sx);                              \
+        int kinc = (sz) * (nxd) * (nyd) - (ny) * (sy) * (nxd);              \
+        PF_UNUSED(nz);                                                      \
+        PF_UNUSED(nzd)
 
 /**
  * @brief Perform a reduction over a BoxLoopI1 iteration space
@@ -89,99 +89,99 @@
  */
 #define BoxLoopReduceI3_default(sum, ...) BoxLoopI3(__VA_ARGS__)
 
-#define BoxLoopI0_default(i, j, k,        \
-                  ix, iy, iz, nx, ny, nz, \
-                  body)                   \
-  {                                       \
-    for (k = iz; k < iz + nz; k++)        \
-    {                                     \
-      for (j = iy; j < iy + ny; j++)      \
-      {                                   \
-        for (i = ix; i < ix + nx; i++)    \
-        {                                 \
-          body;                           \
-        }                                 \
-      }                                   \
-    }                                     \
-  }
+#define BoxLoopI0_default(i, j, k,                \
+                          ix, iy, iz, nx, ny, nz, \
+                          body)                   \
+        {                                         \
+          for (k = iz; k < iz + nz; k++)          \
+          {                                       \
+            for (j = iy; j < iy + ny; j++)        \
+            {                                     \
+              for (i = ix; i < ix + nx; i++)      \
+              {                                   \
+                body;                             \
+              }                                   \
+            }                                     \
+          }                                       \
+        }
 
-#define BoxLoopI1_default(i, j, k,                                              \
-                  ix, iy, iz, nx, ny, nz,                                       \
-                  i1, nx1, ny1, nz1, sx1, sy1, sz1,                             \
-                  body)                                                         \
-  {                                                                             \
-    DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1); \
-    for (k = iz; k < iz + nz; k++)                                              \
-    {                                                                           \
-      for (j = iy; j < iy + ny; j++)                                            \
-      {                                                                         \
-        for (i = ix; i < ix + nx; i++)                                          \
-        {                                                                       \
-          body;                                                                 \
-          i1 += sx1;                                                            \
-        }                                                                       \
-        i1 += PV_jinc_1;                                                        \
-      }                                                                         \
-      i1 += PV_kinc_1;                                                          \
-    }                                                                           \
-  }
+#define BoxLoopI1_default(i, j, k,                                                      \
+                          ix, iy, iz, nx, ny, nz,                                       \
+                          i1, nx1, ny1, nz1, sx1, sy1, sz1,                             \
+                          body)                                                         \
+        {                                                                               \
+          DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);   \
+          for (k = iz; k < iz + nz; k++)                                                \
+          {                                                                             \
+            for (j = iy; j < iy + ny; j++)                                              \
+            {                                                                           \
+              for (i = ix; i < ix + nx; i++)                                            \
+              {                                                                         \
+                body;                                                                   \
+                i1 += sx1;                                                              \
+              }                                                                         \
+              i1 += PV_jinc_1;                                                          \
+            }                                                                           \
+            i1 += PV_kinc_1;                                                            \
+          }                                                                             \
+        }
 
-#define BoxLoopI2_default(i, j, k,                                              \
-                  ix, iy, iz, nx, ny, nz,                                       \
-                  i1, nx1, ny1, nz1, sx1, sy1, sz1,                             \
-                  i2, nx2, ny2, nz2, sx2, sy2, sz2,                             \
-                  body)                                                         \
-  {                                                                             \
-    DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1); \
-    DeclareInc(PV_jinc_2, PV_kinc_2, nx, ny, nz, nx2, ny2, nz2, sx2, sy2, sz2); \
-    for (k = iz; k < iz + nz; k++)                                              \
-    {                                                                           \
-      for (j = iy; j < iy + ny; j++)                                            \
-      {                                                                         \
-        for (i = ix; i < ix + nx; i++)                                          \
-        {                                                                       \
-          body;                                                                 \
-          i1 += sx1;                                                            \
-          i2 += sx2;                                                            \
-        }                                                                       \
-        i1 += PV_jinc_1;                                                        \
-        i2 += PV_jinc_2;                                                        \
-      }                                                                         \
-      i1 += PV_kinc_1;                                                          \
-      i2 += PV_kinc_2;                                                          \
-    }                                                                           \
-  }
+#define BoxLoopI2_default(i, j, k,                                                      \
+                          ix, iy, iz, nx, ny, nz,                                       \
+                          i1, nx1, ny1, nz1, sx1, sy1, sz1,                             \
+                          i2, nx2, ny2, nz2, sx2, sy2, sz2,                             \
+                          body)                                                         \
+        {                                                                               \
+          DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);   \
+          DeclareInc(PV_jinc_2, PV_kinc_2, nx, ny, nz, nx2, ny2, nz2, sx2, sy2, sz2);   \
+          for (k = iz; k < iz + nz; k++)                                                \
+          {                                                                             \
+            for (j = iy; j < iy + ny; j++)                                              \
+            {                                                                           \
+              for (i = ix; i < ix + nx; i++)                                            \
+              {                                                                         \
+                body;                                                                   \
+                i1 += sx1;                                                              \
+                i2 += sx2;                                                              \
+              }                                                                         \
+              i1 += PV_jinc_1;                                                          \
+              i2 += PV_jinc_2;                                                          \
+            }                                                                           \
+            i1 += PV_kinc_1;                                                            \
+            i2 += PV_kinc_2;                                                            \
+          }                                                                             \
+        }
 
-#define BoxLoopI3_default(i, j, k,                                              \
-                  ix, iy, iz, nx, ny, nz,                                       \
-                  i1, nx1, ny1, nz1, sx1, sy1, sz1,                             \
-                  i2, nx2, ny2, nz2, sx2, sy2, sz2,                             \
-                  i3, nx3, ny3, nz3, sx3, sy3, sz3,                             \
-                  body)                                                         \
-  {                                                                             \
-    DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1); \
-    DeclareInc(PV_jinc_2, PV_kinc_2, nx, ny, nz, nx2, ny2, nz2, sx2, sy2, sz2); \
-    DeclareInc(PV_jinc_3, PV_kinc_3, nx, ny, nz, nx3, ny3, nz3, sx3, sy3, sz3); \
-    for (k = iz; k < iz + nz; k++)                                              \
-    {                                                                           \
-      for (j = iy; j < iy + ny; j++)                                            \
-      {                                                                         \
-        for (i = ix; i < ix + nx; i++)                                          \
-        {                                                                       \
-          body;                                                                 \
-          i1 += sx1;                                                            \
-          i2 += sx2;                                                            \
-          i3 += sx3;                                                            \
-        }                                                                       \
-        i1 += PV_jinc_1;                                                        \
-        i2 += PV_jinc_2;                                                        \
-        i3 += PV_jinc_3;                                                        \
-      }                                                                         \
-      i1 += PV_kinc_1;                                                          \
-      i2 += PV_kinc_2;                                                          \
-      i3 += PV_kinc_3;                                                          \
-    }                                                                           \
-  }
+#define BoxLoopI3_default(i, j, k,                                                      \
+                          ix, iy, iz, nx, ny, nz,                                       \
+                          i1, nx1, ny1, nz1, sx1, sy1, sz1,                             \
+                          i2, nx2, ny2, nz2, sx2, sy2, sz2,                             \
+                          i3, nx3, ny3, nz3, sx3, sy3, sz3,                             \
+                          body)                                                         \
+        {                                                                               \
+          DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);   \
+          DeclareInc(PV_jinc_2, PV_kinc_2, nx, ny, nz, nx2, ny2, nz2, sx2, sy2, sz2);   \
+          DeclareInc(PV_jinc_3, PV_kinc_3, nx, ny, nz, nx3, ny3, nz3, sx3, sy3, sz3);   \
+          for (k = iz; k < iz + nz; k++)                                                \
+          {                                                                             \
+            for (j = iy; j < iy + ny; j++)                                              \
+            {                                                                           \
+              for (i = ix; i < ix + nx; i++)                                            \
+              {                                                                         \
+                body;                                                                   \
+                i1 += sx1;                                                              \
+                i2 += sx2;                                                              \
+                i3 += sx3;                                                              \
+              }                                                                         \
+              i1 += PV_jinc_1;                                                          \
+              i2 += PV_jinc_2;                                                          \
+              i3 += PV_jinc_3;                                                          \
+            }                                                                           \
+            i1 += PV_kinc_1;                                                            \
+            i2 += PV_kinc_2;                                                            \
+            i3 += PV_kinc_3;                                                            \
+          }                                                                             \
+        }
 
 /******************************************************************************
 *     SPECIAL NOTE! SPECIAL NOTE! SPECIAL NOTE! SPECIAL NOTE! SPECIAL NOTE!   *
@@ -198,67 +198,67 @@
 *   it's supposed to be and try it out.  Otherwise your stuck.                *
 ******************************************************************************/
 
-#define BoxLoopI4(i, j, k,                                                \
-                  ix, iy, iz, nx, ny, nz,                                 \
-                  i1, nx1, ny1, nz1,                                      \
-                  i2, nx2, ny2, nz2,                                      \
-                  i3, nx3, ny3, nz3,                                      \
-                  i4, nx4, ny4, nz4,                                      \
-                  body)                                                   \
-  {                                                                       \
-    DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, 1, 1, 1); \
-    DeclareInc(PV_jinc_2, PV_kinc_2, nx, ny, nz, nx2, ny2, nz2, 1, 1, 1); \
-    DeclareInc(PV_jinc_3, PV_kinc_3, nx, ny, nz, nx3, ny3, nz3, 1, 1, 1); \
-    DeclareInc(PV_jinc_4, PV_kinc_4, nx, ny, nz, nx4, ny4, nz4, 1, 1, 1); \
-    for (k = iz; k < iz + nz; k++)                                        \
-    {                                                                     \
-      for (j = iy; j < iy + ny; j++)                                      \
-      {                                                                   \
-        for (i = ix; i < ix + nx; i++)                                    \
-        {                                                                 \
-          body;                                                           \
-          i1 += 1;                                                        \
-          i2 += 1;                                                        \
-          i3 += 1;                                                        \
-          i4 += 1;                                                        \
-        }                                                                 \
-        i1 += PV_jinc_1;                                                  \
-        i2 += PV_jinc_2;                                                  \
-        i3 += PV_jinc_3;                                                  \
-        i4 += PV_jinc_4;                                                  \
-      }                                                                   \
-      i1 += PV_kinc_1;                                                    \
-      i2 += PV_kinc_2;                                                    \
-      i3 += PV_kinc_3;                                                    \
-      i4 += PV_kinc_4;                                                    \
-    }                                                                     \
-  }
+#define BoxLoopI4(i, j, k,                                                      \
+                  ix, iy, iz, nx, ny, nz,                                       \
+                  i1, nx1, ny1, nz1,                                            \
+                  i2, nx2, ny2, nz2,                                            \
+                  i3, nx3, ny3, nz3,                                            \
+                  i4, nx4, ny4, nz4,                                            \
+                  body)                                                         \
+        {                                                                       \
+          DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, 1, 1, 1); \
+          DeclareInc(PV_jinc_2, PV_kinc_2, nx, ny, nz, nx2, ny2, nz2, 1, 1, 1); \
+          DeclareInc(PV_jinc_3, PV_kinc_3, nx, ny, nz, nx3, ny3, nz3, 1, 1, 1); \
+          DeclareInc(PV_jinc_4, PV_kinc_4, nx, ny, nz, nx4, ny4, nz4, 1, 1, 1); \
+          for (k = iz; k < iz + nz; k++)                                        \
+          {                                                                     \
+            for (j = iy; j < iy + ny; j++)                                      \
+            {                                                                   \
+              for (i = ix; i < ix + nx; i++)                                    \
+              {                                                                 \
+                body;                                                           \
+                i1 += 1;                                                        \
+                i2 += 1;                                                        \
+                i3 += 1;                                                        \
+                i4 += 1;                                                        \
+              }                                                                 \
+              i1 += PV_jinc_1;                                                  \
+              i2 += PV_jinc_2;                                                  \
+              i3 += PV_jinc_3;                                                  \
+              i4 += PV_jinc_4;                                                  \
+            }                                                                   \
+            i1 += PV_kinc_1;                                                    \
+            i2 += PV_kinc_2;                                                    \
+            i3 += PV_kinc_3;                                                    \
+            i4 += PV_kinc_4;                                                    \
+          }                                                                     \
+        }
 
-#define pgs_BoxLoopI2(i, j, k,                \
-                      ix, iy, iz, nx, ny, nz, \
-                      sx, sy, sz,             \
-                      i1, nx1, ny1, nz1,      \
-                      i2, nx2, ny2, nz2,      \
-                      body)                   \
-  {                                           \
-    i1 = 0;                                   \
-    i2 = 0;                                   \
-    for (k = iz; k < nz; k += sz)             \
-    {                                         \
-      for (j = iy; j < ny; j += sy)           \
-      {                                       \
-        for (i = ix; i < nx; i += sx)         \
-        {                                     \
-          body;                               \
-          i1 += sx;                           \
-          i2 += sx;                           \
-        }                                     \
-        i1 += (sy * nx1 - i + ix);            \
-        i2 += (sy * nx2 - i + ix);            \
-      }                                       \
-      i1 = (k - iz + sz) * nx1 * ny1;         \
-      i2 = (k - iz + sz) * nx2 * ny2;         \
-    }                                         \
-  }
+#define pgs_BoxLoopI2(i, j, k,                      \
+                      ix, iy, iz, nx, ny, nz,       \
+                      sx, sy, sz,                   \
+                      i1, nx1, ny1, nz1,            \
+                      i2, nx2, ny2, nz2,            \
+                      body)                         \
+        {                                           \
+          i1 = 0;                                   \
+          i2 = 0;                                   \
+          for (k = iz; k < nz; k += sz)             \
+          {                                         \
+            for (j = iy; j < ny; j += sy)           \
+            {                                       \
+              for (i = ix; i < nx; i += sx)         \
+              {                                     \
+                body;                               \
+                i1 += sx;                           \
+                i2 += sx;                           \
+              }                                     \
+              i1 += (sy * nx1 - i + ix);            \
+              i2 += (sy * nx2 - i + ix);            \
+            }                                       \
+            i1 = (k - iz + sz) * nx1 * ny1;         \
+            i2 = (k - iz + sz) * nx2 * ny2;         \
+          }                                         \
+        }
 
 #endif
diff --git a/pfsimulator/parflow_lib/matdiag_scale.c b/pfsimulator/parflow_lib/matdiag_scale.c
index a4224bbf5..a7b75af90 100644
--- a/pfsimulator/parflow_lib/matdiag_scale.c
+++ b/pfsimulator/parflow_lib/matdiag_scale.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * FILE: matdiag_scale.c
diff --git a/pfsimulator/parflow_lib/matrix.c b/pfsimulator/parflow_lib/matrix.c
index 1f656b3d1..a569eb2b8 100644
--- a/pfsimulator/parflow_lib/matrix.c
+++ b/pfsimulator/parflow_lib/matrix.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Constructors and destructors for matrix structure.
@@ -611,6 +611,7 @@ Matrix          *NewMatrixType(
         MatrixCommPkg(new_matrix) = NewMatrixUpdatePkg(new_matrix, ghost);
 
       break;
+
     default:
       PARFLOW_ERROR("invalid matrix type");
       break;
diff --git a/pfsimulator/parflow_lib/matrix.h b/pfsimulator/parflow_lib/matrix.h
index 1e8d2b1da..269850b09 100644
--- a/pfsimulator/parflow_lib/matrix.h
+++ b/pfsimulator/parflow_lib/matrix.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Header info for Matrix data structures
@@ -124,7 +124,7 @@ typedef struct {
 
 #define SubmatrixData(submatrix) ((submatrix)->data)
 #define SubmatrixStencilData(submatrix, s) \
-  (((submatrix)->data) + ((submatrix)->data_index[s]))
+        (((submatrix)->data) + ((submatrix)->data_index[s]))
 
 #define SubmatrixDataSpace(submatrix)  ((submatrix)->data_space)
 
@@ -140,18 +140,18 @@ typedef struct {
 #define SubmatrixSY(submatrix)   (SubregionSY(SubmatrixDataSpace(submatrix)))
 #define SubmatrixSZ(submatrix)   (SubregionSZ(SubmatrixDataSpace(submatrix)))
 
-#define SubmatrixEltIndex(submatrix, x, y, z)                   \
-  (((x) - SubmatrixIX(submatrix)) / SubmatrixSX(submatrix) +    \
-   (((y) - SubmatrixIY(submatrix)) / SubmatrixSY(submatrix) +   \
-    (((z) - SubmatrixIZ(submatrix)) / SubmatrixSZ(submatrix)) * \
-    SubmatrixNY(submatrix)) *                                   \
-   SubmatrixNX(submatrix))
+#define SubmatrixEltIndex(submatrix, x, y, z)                         \
+        (((x) - SubmatrixIX(submatrix)) / SubmatrixSX(submatrix) +    \
+         (((y) - SubmatrixIY(submatrix)) / SubmatrixSY(submatrix) +   \
+          (((z) - SubmatrixIZ(submatrix)) / SubmatrixSZ(submatrix)) * \
+          SubmatrixNY(submatrix)) *                                   \
+         SubmatrixNX(submatrix))
 
 #define SubmatrixElt(submatrix, s, x, y, z) \
-  (SubmatrixStencilData(submatrix, s) + SubmatrixEltIndex(submatrix, x, y, z))
+        (SubmatrixStencilData(submatrix, s) + SubmatrixEltIndex(submatrix, x, y, z))
 
 #define SubmatrixSize(submatrix) SubmatrixNX((submatrix)) * SubmatrixNY((submatrix)) * \
-  SubmatrixNZ((submatrix))
+        SubmatrixNZ((submatrix))
 
 
 /*--------------------------------------------------------------------------
diff --git a/pfsimulator/parflow_lib/matvec.c b/pfsimulator/parflow_lib/matvec.c
index 688bf5ec1..f5dc2e7a4 100644
--- a/pfsimulator/parflow_lib/matvec.c
+++ b/pfsimulator/parflow_lib/matvec.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Matrix-vector multply routine.
@@ -618,7 +618,7 @@ void            MatvecSubMat(
           });
         }
 
-	/* Now compute matvec contributions from JC */
+        /* Now compute matvec contributions from JC */
         yp = SubvectorData(y_sub);
         xp = SubvectorData(x_sub);
         for (si = 0; si < 5; si++)     /* loop over only c,w,e,s,n */
diff --git a/pfsimulator/parflow_lib/max_field_value.c b/pfsimulator/parflow_lib/max_field_value.c
index 5048cd0c2..7d829057f 100644
--- a/pfsimulator/parflow_lib/max_field_value.c
+++ b/pfsimulator/parflow_lib/max_field_value.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /****************************************************************************
  *
  *****************************************************************************/
@@ -268,32 +268,32 @@ double  MaxPhaseFieldValue(
           tmp_max = fabs(vp[vi]) / psi_max;
           switch (dir)
           {
-            case 0:
-              {
-                if (tmp_max > max_xdir_value)
+              case 0:
                 {
-                  max_xdir_value = tmp_max;
+                  if (tmp_max > max_xdir_value)
+                  {
+                    max_xdir_value = tmp_max;
+                  }
+                  break;
                 }
-                break;
-              }
 
-            case 1:
-              {
-                if (tmp_max > max_ydir_value)
+              case 1:
                 {
-                  max_ydir_value = tmp_max;
+                  if (tmp_max > max_ydir_value)
+                  {
+                    max_ydir_value = tmp_max;
+                  }
+                  break;
                 }
-                break;
-              }
 
-            case 2:
-              {
-                if (tmp_max > max_zdir_value)
+              case 2:
                 {
-                  max_zdir_value = tmp_max;
+                  if (tmp_max > max_zdir_value)
+                  {
+                    max_zdir_value = tmp_max;
+                  }
+                  break;
                 }
-                break;
-              }
           }
         }
       });
@@ -364,7 +364,7 @@ double  MaxTotalFieldValue(
   double f_prime_max, h_prime_max = 0.0;
   double max_field_value, psi_max, ds = 0.0;
   double max_xdir_value, max_ydir_value, max_zdir_value,
-    max_total_value, max_gravity_value;
+         max_total_value, max_gravity_value;
   double tmp, tmp_max, tmp_total = 0.0, tmp_gravity = 0.0;
 
   double a, b, den0, den1, dtmp, g, constant;
@@ -577,40 +577,40 @@ double  MaxTotalFieldValue(
 
           switch (dir)
           {
-            case 0:
-              {
-                if (tmp_max > max_xdir_value)
+              case 0:
                 {
-                  max_xdir_value = tmp_max;
+                  if (tmp_max > max_xdir_value)
+                  {
+                    max_xdir_value = tmp_max;
+                  }
+                  break;
                 }
-                break;
-              }
 
-            case 1:
-              {
-                if (tmp_max > max_ydir_value)
+              case 1:
                 {
-                  max_ydir_value = tmp_max;
+                  if (tmp_max > max_ydir_value)
+                  {
+                    max_ydir_value = tmp_max;
+                  }
+                  break;
                 }
-                break;
-              }
 
-            case 2:
-              {
-                if (tmp_max > max_zdir_value)
+              case 2:
                 {
-                  max_zdir_value = tmp_max;
+                  if (tmp_max > max_zdir_value)
+                  {
+                    max_zdir_value = tmp_max;
+                  }
+                  if (tmp_total > max_total_value)
+                  {
+                    max_total_value = tmp_total;
+                  }
+                  if (tmp_gravity > max_gravity_value)
+                  {
+                    max_gravity_value = tmp_gravity;
+                  }
+                  break;
                 }
-                if (tmp_total > max_total_value)
-                {
-                  max_total_value = tmp_total;
-                }
-                if (tmp_gravity > max_gravity_value)
-                {
-                  max_gravity_value = tmp_gravity;
-                }
-                break;
-              }
           }
         }
       });
diff --git a/pfsimulator/parflow_lib/metadata.c b/pfsimulator/parflow_lib/metadata.c
index 4dcd00d3a..f20f64efb 100644
--- a/pfsimulator/parflow_lib/metadata.c
+++ b/pfsimulator/parflow_lib/metadata.c
@@ -48,14 +48,14 @@ cJSON* js_domains = NULL;
 // Print a message on rank 0 and return 0. Note that ##__VA_ARGS__ is
 // used to allow an empty __VA_ARGS__ , which is not standard but is
 // supported by most major compilers (clang, gcc, msvc).
-#define METADATA_ERROR(message, ...) \
-  { \
-    if (!amps_Rank(amps_CommWorld) && message && message[0]) \
-    { \
-      amps_Printf(message, ##__VA_ARGS__ ); \
-    } \
-    return 0; \
-  }
+#define METADATA_ERROR(message, ...)                               \
+        {                                                          \
+          if (!amps_Rank(amps_CommWorld) && message && message[0]) \
+          {                                                        \
+            amps_Printf(message, ## __VA_ARGS__);                  \
+          }                                                        \
+          return 0;                                                \
+        }
 
 static void MetadataAddParflowBuildInfo(cJSON* pf)
 {
@@ -592,59 +592,59 @@ int MetadataAddForcingField(
   if (!field_name)
   {
     METADATA_ERROR(
-      "Unable to add metadata for null field.\n");
+                   "Unable to add metadata for null field.\n");
   }
   if (num_field_components <= 0)
   {
     METADATA_ERROR(
-      "Unable to add metadata for \"%s\"; invalid components (%d).\n",
-      field_name, num_field_components);
+                   "Unable to add metadata for \"%s\"; invalid components (%d).\n",
+                   field_name, num_field_components);
   }
   if (!clm_metfile)
   {
     METADATA_ERROR(
-      "Unable to add metadata for \"%s\"; no CLM Met file.\n", field_name);
+                   "Unable to add metadata for \"%s\"; no CLM Met file.\n", field_name);
   }
   if (!clm_metpath)
   {
     METADATA_ERROR(
-      "Unable to add metadata for \"%s\"; no CLM Met path.\n", field_name);
+                   "Unable to add metadata for \"%s\"; no CLM Met path.\n", field_name);
   }
   if (!field_placement)
   {
     METADATA_ERROR(
-      "Unable to add metadata for \"%s\"; null field placement.\n",
-      field_name);
+                   "Unable to add metadata for \"%s\"; null field placement.\n",
+                   field_name);
   }
   if (!field_domain)
   {
     METADATA_ERROR(
-      "Unable to add metadata for \"%s\"; null field domain.\n", field_name);
+                   "Unable to add metadata for \"%s\"; null field domain.\n", field_name);
   }
   if (!field_component_postfixes)
   {
     METADATA_ERROR(
-      "Unable to add metadata for \"%s\"; null field component_postfixes.\n",
-      field_name);
+                   "Unable to add metadata for \"%s\"; null field component_postfixes.\n",
+                   field_name);
   }
   if (clm_metforce < 2 || clm_metforce > 3)
   {
     METADATA_ERROR(
-      "Unable to add metadata for \"%s\"; Unhandled CLM Met forcing %d.\n",
-      field_name, clm_metforce);
+                   "Unable to add metadata for \"%s\"; Unhandled CLM Met forcing %d.\n",
+                   field_name, clm_metforce);
   }
   if (clm_metnt <= 0)
   {
     METADATA_ERROR(
-      "Unable to add metadata for \"%s\"; %d timesteps provided (> 0 required).\n",
-      field_name, clm_metnt);
+                   "Unable to add metadata for \"%s\"; %d timesteps provided (> 0 required).\n",
+                   field_name, clm_metnt);
   }
   if (clm_istep_start > clm_metnt)
   {
     METADATA_ERROR(
-      "Unable to add metadata for \"%s\"; "
-      "start time (%d) beyond forcing timesteps provided (%d).\n",
-      field_name, clm_istep_start, clm_metnt);
+                   "Unable to add metadata for \"%s\"; "
+                   "start time (%d) beyond forcing timesteps provided (%d).\n",
+                   field_name, clm_istep_start, clm_metnt);
   }
 
   cJSON* field_item = cJSON_CreateObject();
diff --git a/pfsimulator/parflow_lib/mg_semi.c b/pfsimulator/parflow_lib/mg_semi.c
index 222bb84d1..ea7cd3565 100644
--- a/pfsimulator/parflow_lib/mg_semi.c
+++ b/pfsimulator/parflow_lib/mg_semi.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Multigrid with semi-coarsening strategy.
@@ -96,8 +96,8 @@ void     MGSemi(
                 double  tol,
                 int     zero)
 {
-  PUSH_NVTX("MGSemi",2)
-  
+  PUSH_NVTX("MGSemi", 2)
+
   PFModule      *this_module = ThisPFModule;
   PublicXtra    *public_xtra = (PublicXtra*)PFModulePublicXtra(this_module);
   InstanceXtra  *instance_xtra = (InstanceXtra*)PFModuleInstanceXtra(this_module);
@@ -212,7 +212,7 @@ void     MGSemi(
   /* smooth (use `zero' to determine initial x) */
   PFModuleInvokeType(LinearSolverInvoke, smooth_l[0], (x, b, 0.0, zero));
 
-  PUSH_NVTX("MGSemi_solveloop",4)
+  PUSH_NVTX("MGSemi_solveloop", 4)
   while (++i)
   {
     /*--------------------------------------------------------------------
@@ -772,7 +772,7 @@ PFModule     *MGSemiInitInstanceXtra(
                                      Matrix *     A,
                                      double *     temp_data)
 {
-  PUSH_NVTX("MGSemiInitInstanceXtra",3)
+  PUSH_NVTX("MGSemiInitInstanceXtra", 3)
 
   PFModule      *this_module = ThisPFModule;
   PublicXtra    *public_xtra = (PublicXtra*)PFModulePublicXtra(this_module);
diff --git a/pfsimulator/parflow_lib/mg_semi_prolong.c b/pfsimulator/parflow_lib/mg_semi_prolong.c
index 06f14a812..2cc5f7351 100644
--- a/pfsimulator/parflow_lib/mg_semi_prolong.c
+++ b/pfsimulator/parflow_lib/mg_semi_prolong.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Operator induced prolongation for MGSemi module
diff --git a/pfsimulator/parflow_lib/mg_semi_restrict.c b/pfsimulator/parflow_lib/mg_semi_restrict.c
index 591bb266e..2e59ceed6 100644
--- a/pfsimulator/parflow_lib/mg_semi_restrict.c
+++ b/pfsimulator/parflow_lib/mg_semi_restrict.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Operator induced restriction for MGSemi module
diff --git a/pfsimulator/parflow_lib/n_vector.c b/pfsimulator/parflow_lib/n_vector.c
index 621e0887d..bf2ba95c9 100644
--- a/pfsimulator/parflow_lib/n_vector.c
+++ b/pfsimulator/parflow_lib/n_vector.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
diff --git a/pfsimulator/parflow_lib/n_vector.h b/pfsimulator/parflow_lib/n_vector.h
index 14d21a25e..0b61105d5 100644
--- a/pfsimulator/parflow_lib/n_vector.h
+++ b/pfsimulator/parflow_lib/n_vector.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef _N_VECTOR_HEADER
 #define _N_VECTOR_HEADER
diff --git a/pfsimulator/parflow_lib/new_endpts.c b/pfsimulator/parflow_lib/new_endpts.c
index 750b2b045..e6dd09d1b 100644
--- a/pfsimulator/parflow_lib/new_endpts.c
+++ b/pfsimulator/parflow_lib/new_endpts.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
  *
  * File:        NewEndpts.c
diff --git a/pfsimulator/parflow_lib/nl_function_eval.c b/pfsimulator/parflow_lib/nl_function_eval.c
index 8016095d0..47a03ec02 100644
--- a/pfsimulator/parflow_lib/nl_function_eval.c
+++ b/pfsimulator/parflow_lib/nl_function_eval.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 #include "llnlmath.h"
@@ -123,7 +123,7 @@ void NlFunctionEval(Vector *     pressure, /* Current pressure values */
                     Vector *     y_velocity,
                     Vector *     z_velocity)
 {
-  PUSH_NVTX("NlFunctionEval",0)
+  PUSH_NVTX("NlFunctionEval", 0)
 
   PFModule      *this_module = ThisPFModule;
   InstanceXtra  *instance_xtra = (InstanceXtra*)PFModuleInstanceXtra(this_module);
@@ -180,7 +180,7 @@ void NlFunctionEval(Vector *     pressure, /* Current pressure values */
   Vector      *FBy = ProblemDataFBy(problem_data);
   Vector      *FBz = ProblemDataFBz(problem_data);
   Subvector   *FBx_sub, *FBy_sub, *FBz_sub;  //@RMM
-  double      *FBx_dat=NULL, *FBy_dat=NULL, *FBz_dat=NULL;   //@RMM
+  double      *FBx_dat = NULL, *FBy_dat = NULL, *FBz_dat = NULL;   //@RMM
 
 
   double gravity = ProblemGravity(problem);
@@ -190,7 +190,7 @@ void NlFunctionEval(Vector *     pressure, /* Current pressure values */
 
   Subvector   *p_sub, *d_sub, *od_sub, *s_sub, *os_sub, *po_sub, *op_sub, *ss_sub, *et_sub;
   Subvector   *f_sub, *rp_sub, *permx_sub, *permy_sub, *permz_sub;
-  
+
   Grid        *grid = VectorGrid(pressure);
   Grid        *grid2d = VectorGrid(x_sl);
 
@@ -222,7 +222,6 @@ void NlFunctionEval(Vector *     pressure, /* Current pressure values */
   VectorUpdateCommHandle  *handle;
 
   BeginTiming(public_xtra->time_index);
-
   /* Initialize function values to zero. */
   PFVConstInit(0.0, fval);
 
@@ -516,21 +515,21 @@ void NlFunctionEval(Vector *     pressure, /* Current pressure values */
       ForPatchCellsPerFace(DirichletBC,
                            BeforeAllCells(DoNothing),
                            LoopVars(i, j, k, ival, bc_struct, ipatch, is),
-                           Locals(int pp_idx, ip; double value;),
+                           Locals(int pp_idx, ip; double value; ),
                            CellSetup({
-                               pp_idx = 0;
-                               ip = SubvectorEltIndex(p_sub, i, j, k);
-                               value = bc_patch_values[ival];
-                             }),
-                           FACE(LeftFace,    { pp_idx = ip - 1; }),
-                           FACE(RightFace,   { pp_idx = ip + 1; }),
-                           FACE(DownFace,    { pp_idx = ip - sy_p; }),
-                           FACE(UpFace,      { pp_idx = ip + sy_p; }),
-                           FACE(BackFace,    { pp_idx = ip - sz_p; }),
-                           FACE(FrontFace,   { pp_idx = ip + sz_p; }),
-                           CellFinalize( { pp[pp_idx] = value; }),
+        pp_idx = 0;
+        ip = SubvectorEltIndex(p_sub, i, j, k);
+        value = bc_patch_values[ival];
+      }),
+                           FACE(LeftFace, { pp_idx = ip - 1; }),
+                           FACE(RightFace, { pp_idx = ip + 1; }),
+                           FACE(DownFace, { pp_idx = ip - sy_p; }),
+                           FACE(UpFace, { pp_idx = ip + sy_p; }),
+                           FACE(BackFace, { pp_idx = ip - sz_p; }),
+                           FACE(FrontFace, { pp_idx = ip + sz_p; }),
+                           CellFinalize({ pp[pp_idx] = value; }),
                            AfterAllCells(DoNothing)
-        );     /* End DirichletBC */
+                           ); /* End DirichletBC */
     }          /* End ipatch loop */
   }            /* End subgrid loop */
 
@@ -642,44 +641,44 @@ void NlFunctionEval(Vector *     pressure, /* Current pressure values */
       double del_y_slope = 1.0;
 
       double x_dir_g = NAN;
-      double x_dir_g_c= NAN;
-      double y_dir_g= NAN;
-      double y_dir_g_c= NAN;
+      double x_dir_g_c = NAN;
+      double y_dir_g = NAN;
+      double y_dir_g_c = NAN;
 
 //@RMM  tfgupwind == 0 (default) should give original behavior
 // tfgupwind 1 should still use sine but upwind
 // tfgupwdin 2 just upwind
       switch (public_xtra->tfgupwind)
       {
-        case 0:
-          {
-            // default formulation in Maxwell 2013
-            x_dir_g = Mean(gravity * sin(atan(x_ssl_dat[io])), gravity * sin(atan(x_ssl_dat[io + 1])));
-            x_dir_g_c = Mean(gravity * cos(atan(x_ssl_dat[io])), gravity * cos(atan(x_ssl_dat[io + 1])));
-            y_dir_g = Mean(gravity * sin(atan(y_ssl_dat[io])), gravity * sin(atan(y_ssl_dat[io + sy_p])));
-            y_dir_g_c = Mean(gravity * cos(atan(y_ssl_dat[io])), gravity * cos(atan(y_ssl_dat[io + sy_p])));
-            break;
-          }
-
-        case 1:
-          {
-            // direct upwinding, no averaging with sines
-            x_dir_g = gravity * sin(atan(x_ssl_dat[io]));
-            x_dir_g_c = gravity * cos(atan(x_ssl_dat[io]));
-            y_dir_g = gravity * sin(atan(y_ssl_dat[io]));
-            y_dir_g_c = gravity * cos(atan(y_ssl_dat[io]));
-            break;
-          }
-
-        case 2:
-          {
-            // direct upwinding, no averaging no sines
-            x_dir_g = x_ssl_dat[io];
-            x_dir_g_c = 1.0;
-            y_dir_g = y_ssl_dat[io];
-            y_dir_g_c = 1.0;
-            break;
-          }
+          case 0:
+            {
+              // default formulation in Maxwell 2013
+              x_dir_g = Mean(gravity * sin(atan(x_ssl_dat[io])), gravity * sin(atan(x_ssl_dat[io + 1])));
+              x_dir_g_c = Mean(gravity * cos(atan(x_ssl_dat[io])), gravity * cos(atan(x_ssl_dat[io + 1])));
+              y_dir_g = Mean(gravity * sin(atan(y_ssl_dat[io])), gravity * sin(atan(y_ssl_dat[io + sy_p])));
+              y_dir_g_c = Mean(gravity * cos(atan(y_ssl_dat[io])), gravity * cos(atan(y_ssl_dat[io + sy_p])));
+              break;
+            }
+
+          case 1:
+            {
+              // direct upwinding, no averaging with sines
+              x_dir_g = gravity * sin(atan(x_ssl_dat[io]));
+              x_dir_g_c = gravity * cos(atan(x_ssl_dat[io]));
+              y_dir_g = gravity * sin(atan(y_ssl_dat[io]));
+              y_dir_g_c = gravity * cos(atan(y_ssl_dat[io]));
+              break;
+            }
+
+          case 2:
+            {
+              // direct upwinding, no averaging no sines
+              x_dir_g = x_ssl_dat[io];
+              x_dir_g_c = 1.0;
+              y_dir_g = y_ssl_dat[io];
+              y_dir_g_c = 1.0;
+              break;
+            }
       }
       /* Calculate right face velocity.
        * diff_l >= 0 implies flow goes left to right */
@@ -688,12 +687,12 @@ void NlFunctionEval(Vector *     pressure, /* Current pressure values */
       double updir = (diff_l / dx) * x_dir_g_c - x_dir_g;
 
       double u_right = z_mult_dat[ip] * ffx * del_y_slope * PMean(pp[ip], pp[ip + 1],
-                                                           permxp[ip], permxp[ip + 1])
-                * (diff_l / (dx * del_x_slope)) * x_dir_g_c
-                * RPMean(updir, 0.0,
-                         rpp[ip] * dp[ip],
-                         rpp[ip + 1] * dp[ip + 1])
-                / viscosity;
+                                                                  permxp[ip], permxp[ip + 1])
+                       * (diff_l / (dx * del_x_slope)) * x_dir_g_c
+                       * RPMean(updir, 0.0,
+                                rpp[ip] * dp[ip],
+                                rpp[ip + 1] * dp[ip + 1])
+                       / viscosity;
 
       /* Calculate right face velocity gravity terms
        * @RMM added sin* g term to test terrain-following grid
@@ -714,12 +713,12 @@ void NlFunctionEval(Vector *     pressure, /* Current pressure values */
       updir = (diff_l / dy) * y_dir_g_c - y_dir_g;
 
       double u_front = z_mult_dat[ip] * ffy * del_x_slope
-                * PMean(pp[ip], pp[ip + sy_p], permyp[ip], permyp[ip + sy_p])
-                * (diff_l / (dy * del_y_slope)) * y_dir_g_c
-                * RPMean(updir, 0.0,
-                         rpp[ip] * dp[ip],
-                         rpp[ip + sy_p] * dp[ip + sy_p])
-                / viscosity;
+                       * PMean(pp[ip], pp[ip + sy_p], permyp[ip], permyp[ip + sy_p])
+                       * (diff_l / (dy * del_y_slope)) * y_dir_g_c
+                       * RPMean(updir, 0.0,
+                                rpp[ip] * dp[ip],
+                                rpp[ip + sy_p] * dp[ip + sy_p])
+                       / viscosity;
 
       /* Calculate front face velocity gravity terms
        * @RMM added sin* g term to test terrain-following grid
@@ -741,22 +740,22 @@ void NlFunctionEval(Vector *     pressure, /* Current pressure values */
 
 
       double lower_cond_l = pp[ip] / sep_l
-                   - (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p]))
-                   * dp[ip] * gravity * z_dir_g;
+                            - (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p]))
+                            * dp[ip] * gravity * z_dir_g;
 
       double upper_cond_l = pp[ip + sz_p] / sep_l
-                   + (z_mult_dat[ip + sz_p] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p]))
-                   * dp[ip + sz_p] * gravity * z_dir_g;
+                            + (z_mult_dat[ip + sz_p] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p]))
+                            * dp[ip + sz_p] * gravity * z_dir_g;
 
 
       diff_l = (lower_cond_l - upper_cond_l);
 
       double u_upper = ffz * del_x_slope * del_y_slope
-                * PMeanDZ(permzp[ip], permzp[ip + sz_p], z_mult_dat[ip], z_mult_dat[ip + sz_p])
-                * diff_l
-                * RPMean(lower_cond_l, upper_cond_l, rpp[ip] * dp[ip],
-                         rpp[ip + sz_p] * dp[ip + sz_p])
-                / viscosity;
+                       * PMeanDZ(permzp[ip], permzp[ip + sz_p], z_mult_dat[ip], z_mult_dat[ip + sz_p])
+                       * diff_l
+                       * RPMean(lower_cond_l, upper_cond_l, rpp[ip] * dp[ip],
+                                rpp[ip + sz_p] * dp[ip + sz_p])
+                       / viscosity;
 
 /*  add in flow barrier values
  * assumes that ip is the cell face between ip and ip+1
@@ -889,252 +888,252 @@ void NlFunctionEval(Vector *     pressure, /* Current pressure values */
                                   double diff, u_new, u_old, value, h;
                                   double x_dir_g, y_dir_g, z_dir_g;
                                   double sep, lower_cond, upper_cond;
-                                  double del_x_slope, del_y_slope;),
+                                  double del_x_slope, del_y_slope; ),
                            CellSetup(
-                           {
-                             dir = 0;
-                             ip = SubvectorEltIndex(p_sub, i, j, k);
-
-                             vel_idx = 0;
-                             vx_l = SubvectorEltIndex(vx_sub, i, j, k);
-                             vy_l = SubvectorEltIndex(vy_sub, i, j, k);
-                             vz_l = SubvectorEltIndex(vz_sub, i, j, k);
-                             vel_vec = NULL;
-                             h = 0.0;
-
-                             diff = 0.0e0;
-                             u_new = 0.0e0;
-                             u_old = 0.0e0;
-
-                             value = bc_patch_values[ival];
-                             x_dir_g = 0.0;
-                             y_dir_g = 0.0;
-                             z_dir_g = 1.0;
-
-                             sep = 0.0;
-                             lower_cond = 0.0;
-                             upper_cond = 0.0;
-                             // del_x_slope = (1.0 / cos(atan(x_ssl_dat[io])));
-                             // del_y_slope = (1.0 / cos(atan(y_ssl_dat[io])));
-
-                             del_x_slope = 1.0;
-                             del_y_slope = 1.0;
-                           }),
+      {
+        dir = 0;
+        ip = SubvectorEltIndex(p_sub, i, j, k);
+
+        vel_idx = 0;
+        vx_l = SubvectorEltIndex(vx_sub, i, j, k);
+        vy_l = SubvectorEltIndex(vy_sub, i, j, k);
+        vz_l = SubvectorEltIndex(vz_sub, i, j, k);
+        vel_vec = NULL;
+        h = 0.0;
+
+        diff = 0.0e0;
+        u_new = 0.0e0;
+        u_old = 0.0e0;
+
+        value = bc_patch_values[ival];
+        x_dir_g = 0.0;
+        y_dir_g = 0.0;
+        z_dir_g = 1.0;
+
+        sep = 0.0;
+        lower_cond = 0.0;
+        upper_cond = 0.0;
+        // del_x_slope = (1.0 / cos(atan(x_ssl_dat[io])));
+        // del_y_slope = (1.0 / cos(atan(y_ssl_dat[io])));
+
+        del_x_slope = 1.0;
+        del_y_slope = 1.0;
+      }),
                            FACE(LeftFace,
-                           {
-                             h = ffx * z_mult_dat[ip] * del_y_slope;
-                             vel_vec = vx;
-                             vel_idx = vx_l;
-
-                             dir = -1;
-                             diff = pp[ip - 1] - pp[ip];
-                             u_old = h
-                                     * PMean(pp[ip - 1], pp[ip], permxp[ip - 1], permxp[ip])
-                                     * (diff / dx * del_x_slope)
-                                     * RPMean(pp[ip - 1], pp[ip],
-                                              rpp[ip - 1] * dp[ip - 1], rpp[ip] * dp[ip])
-                                     / viscosity;
-
-                             u_old += h *
-                                      PMean(pp[ip - 1], pp[ip],
-                                            permxp[ip - 1], permxp[ip])
-                                      * (-x_dir_g)
-                                      * RPMean(pp[ip - 1], pp[ip], rpp[ip - 1] * dp[ip - 1],
-                                               rpp[ip] * dp[ip])
-                                      / viscosity;
-
-                             diff = value - pp[ip];
-                             u_new = RPMean(value, pp[ip],
-                                            rpp[ip - 1] * dp[ip - 1], rpp[ip] * dp[ip]);
-                             u_new = u_new * h
-                                     * (permxp[ip] / viscosity)
-                                     * 2.0 * (diff / dx);
-                           }),
+      {
+        h = ffx * z_mult_dat[ip] * del_y_slope;
+        vel_vec = vx;
+        vel_idx = vx_l;
+
+        dir = -1;
+        diff = pp[ip - 1] - pp[ip];
+        u_old = h
+                * PMean(pp[ip - 1], pp[ip], permxp[ip - 1], permxp[ip])
+                * (diff / dx * del_x_slope)
+                * RPMean(pp[ip - 1], pp[ip],
+                         rpp[ip - 1] * dp[ip - 1], rpp[ip] * dp[ip])
+                / viscosity;
+
+        u_old += h *
+                 PMean(pp[ip - 1], pp[ip],
+                       permxp[ip - 1], permxp[ip])
+                 * (-x_dir_g)
+                 * RPMean(pp[ip - 1], pp[ip], rpp[ip - 1] * dp[ip - 1],
+                          rpp[ip] * dp[ip])
+                 / viscosity;
+
+        diff = value - pp[ip];
+        u_new = RPMean(value, pp[ip],
+                       rpp[ip - 1] * dp[ip - 1], rpp[ip] * dp[ip]);
+        u_new = u_new * h
+                * (permxp[ip] / viscosity)
+                * 2.0 * (diff / dx);
+      }),
                            FACE(RightFace,
-                           {
-                             h = ffx * z_mult_dat[ip] * del_y_slope;
-                             vel_vec = vx;
-                             vel_idx = vx_l + 1;
-
-                             dir = 1;
-                             diff = pp[ip] - pp[ip + 1];
-                             u_old = h
-                                     * PMean(pp[ip], pp[ip + 1], permxp[ip], permxp[ip + 1])
-                                     * (diff / dx * del_x_slope)
-                                     * RPMean(pp[ip], pp[ip + 1],
-                                              rpp[ip] * dp[ip], rpp[ip + 1] * dp[ip + 1])
-                                     / viscosity;
-
-                             u_old += h
-                                      * PMean(pp[ip], pp[ip + 1],
-                                              permxp[ip], permxp[ip + 1])
-                                      * (-x_dir_g)
-                                      * RPMean(pp[ip], pp[ip + 1], rpp[ip] * dp[ip],
-                                               rpp[ip + 1] * dp[ip + 1])
-                                      / viscosity;
-
-                             diff = pp[ip] - value;
-                             u_new = RPMean(pp[ip], value,
-                                            rpp[ip] * dp[ip], rpp[ip + 1] * dp[ip + 1]);
-                             u_new = u_new * h
-                                     * (permxp[ip] / viscosity)
-                                     * 2.0 * (diff / dx);
-                           }),
+      {
+        h = ffx * z_mult_dat[ip] * del_y_slope;
+        vel_vec = vx;
+        vel_idx = vx_l + 1;
+
+        dir = 1;
+        diff = pp[ip] - pp[ip + 1];
+        u_old = h
+                * PMean(pp[ip], pp[ip + 1], permxp[ip], permxp[ip + 1])
+                * (diff / dx * del_x_slope)
+                * RPMean(pp[ip], pp[ip + 1],
+                         rpp[ip] * dp[ip], rpp[ip + 1] * dp[ip + 1])
+                / viscosity;
+
+        u_old += h
+                 * PMean(pp[ip], pp[ip + 1],
+                         permxp[ip], permxp[ip + 1])
+                 * (-x_dir_g)
+                 * RPMean(pp[ip], pp[ip + 1], rpp[ip] * dp[ip],
+                          rpp[ip + 1] * dp[ip + 1])
+                 / viscosity;
+
+        diff = pp[ip] - value;
+        u_new = RPMean(pp[ip], value,
+                       rpp[ip] * dp[ip], rpp[ip + 1] * dp[ip + 1]);
+        u_new = u_new * h
+                * (permxp[ip] / viscosity)
+                * 2.0 * (diff / dx);
+      }),
                            FACE(DownFace,
-                           {
-                             h = ffy * z_mult_dat[ip] * del_x_slope;
-                             vel_vec = vy;
-                             vel_idx = vy_l;
-
-                             dir = -1;
-                             diff = pp[ip - sy_p] - pp[ip];
-                             u_old = h
-                                     * PMean(pp[ip - sy_p], pp[ip],
-                                             permyp[ip - sy_p], permyp[ip])
-                                     * (diff / dy * del_y_slope)
-                                     * RPMean(pp[ip - sy_p], pp[ip],
-                                              rpp[ip - sy_p] * dp[ip - sy_p], rpp[ip] * dp[ip])
-                                     / viscosity;
-
-                             u_old += h *
-                                      PMean(pp[ip], pp[ip - sy_p], permyp[ip],
-                                            permyp[ip - sy_p])
-                                      * (-y_dir_g)
-                                      * RPMean(pp[ip], pp[ip - sy_p], rpp[ip] * dp[ip],
-                                               rpp[ip - sy_p] * dp[ip - sy_p])
-                                      / viscosity;
-
-
-                             diff = value - pp[ip];
-                             u_new = RPMean(value, pp[ip],
-                                            rpp[ip - sy_p] * dp[ip - sy_p], rpp[ip] * dp[ip]);
-                             u_new = u_new * h * (permyp[ip] / viscosity)
-                                     * 2.0 * (diff / dy);
-                           }),
+      {
+        h = ffy * z_mult_dat[ip] * del_x_slope;
+        vel_vec = vy;
+        vel_idx = vy_l;
+
+        dir = -1;
+        diff = pp[ip - sy_p] - pp[ip];
+        u_old = h
+                * PMean(pp[ip - sy_p], pp[ip],
+                        permyp[ip - sy_p], permyp[ip])
+                * (diff / dy * del_y_slope)
+                * RPMean(pp[ip - sy_p], pp[ip],
+                         rpp[ip - sy_p] * dp[ip - sy_p], rpp[ip] * dp[ip])
+                / viscosity;
+
+        u_old += h *
+                 PMean(pp[ip], pp[ip - sy_p], permyp[ip],
+                       permyp[ip - sy_p])
+                 * (-y_dir_g)
+                 * RPMean(pp[ip], pp[ip - sy_p], rpp[ip] * dp[ip],
+                          rpp[ip - sy_p] * dp[ip - sy_p])
+                 / viscosity;
+
+
+        diff = value - pp[ip];
+        u_new = RPMean(value, pp[ip],
+                       rpp[ip - sy_p] * dp[ip - sy_p], rpp[ip] * dp[ip]);
+        u_new = u_new * h * (permyp[ip] / viscosity)
+                * 2.0 * (diff / dy);
+      }),
                            FACE(UpFace,
-                           {
-                             h = ffy * z_mult_dat[ip] * del_x_slope;
-                             vel_vec = vy;
-                             vel_idx = vy_l + sy_v;
-
-                             dir = 1;
-                             diff = pp[ip] - pp[ip + sy_p];
-                             u_old = h
-                                     * PMean(pp[ip], pp[ip + sy_p],
-                                             permyp[ip], permyp[ip + sy_p])
-                                     * (diff / dy * del_y_slope)
-                                     * RPMean(pp[ip], pp[ip + sy_p],
-                                              rpp[ip] * dp[ip], rpp[ip + sy_p] * dp[ip + sy_p])
-                                     / viscosity;
-
-                             u_old += h
-                                      * PMean(pp[ip], pp[ip + sy_p], permyp[ip],
-                                              permyp[ip + sy_p])
-                                      * (-y_dir_g)
-                                      * RPMean(pp[ip], pp[ip + sy_p], rpp[ip] * dp[ip],
-                                               rpp[ip + sy_p] * dp[ip + sy_p])
-                                      / viscosity;
-
-
-                             diff = pp[ip] - value;
-                             u_new = RPMean(pp[ip], value,
-                                            rpp[ip] * dp[ip], rpp[ip + sy_p] * dp[ip + sy_p]);
-                             u_new = u_new * h * (permyp[ip] / viscosity)
-                                     * 2.0 * (diff / dy);
-                           }),
+      {
+        h = ffy * z_mult_dat[ip] * del_x_slope;
+        vel_vec = vy;
+        vel_idx = vy_l + sy_v;
+
+        dir = 1;
+        diff = pp[ip] - pp[ip + sy_p];
+        u_old = h
+                * PMean(pp[ip], pp[ip + sy_p],
+                        permyp[ip], permyp[ip + sy_p])
+                * (diff / dy * del_y_slope)
+                * RPMean(pp[ip], pp[ip + sy_p],
+                         rpp[ip] * dp[ip], rpp[ip + sy_p] * dp[ip + sy_p])
+                / viscosity;
+
+        u_old += h
+                 * PMean(pp[ip], pp[ip + sy_p], permyp[ip],
+                         permyp[ip + sy_p])
+                 * (-y_dir_g)
+                 * RPMean(pp[ip], pp[ip + sy_p], rpp[ip] * dp[ip],
+                          rpp[ip + sy_p] * dp[ip + sy_p])
+                 / viscosity;
+
+
+        diff = pp[ip] - value;
+        u_new = RPMean(pp[ip], value,
+                       rpp[ip] * dp[ip], rpp[ip + sy_p] * dp[ip + sy_p]);
+        u_new = u_new * h * (permyp[ip] / viscosity)
+                * 2.0 * (diff / dy);
+      }),
                            FACE(BackFace,
-                           {
-                             h = ffz * del_x_slope * del_y_slope;
-                             vel_vec = vz;
-                             vel_idx = vz_l;
-
-                             dir = -1;
-                             sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_p]); //RMM
-
-                             lower_cond = pp[ip - sz_p] / sep
-                                          - (z_mult_dat[ip - sz_p] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip - sz_p] * gravity *
-                                          z_dir_g;
-
-                             upper_cond = pp[ip] / sep + (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip] * gravity *
-                                          z_dir_g;
-
-                             diff = (lower_cond - upper_cond);
-
-                             u_old = h
-                                     * PMeanDZ(permzp[ip - sz_p], permzp[ip],
-                                               z_mult_dat[ip - sz_p], z_mult_dat[ip])
-                                     * diff
-                                     * RPMean(lower_cond, upper_cond,
-                                              rpp[ip - sz_p] * dp[ip - sz_p], rpp[ip] * dp[ip])
-                                     / viscosity;
-
-                             sep = dz * z_mult_dat[ip];
-
-                             lower_cond = value / sep - 0.25 * dp[ip] * gravity;
-                             upper_cond = pp[ip] / sep + 0.25 * dp[ip] * gravity;
-                             diff = (lower_cond - upper_cond);
-                             u_new = RPMean(lower_cond, upper_cond,
-                                            rpp[ip - sz_p] * dp[ip - sz_p], rpp[ip] * dp[ip]);
-                             u_new = u_new * h *
-                                     (permzp[ip] / viscosity)
-                                     * 2.0 * diff;
-                           }),
+      {
+        h = ffz * del_x_slope * del_y_slope;
+        vel_vec = vz;
+        vel_idx = vz_l;
+
+        dir = -1;
+        sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_p]);                      //RMM
+
+        lower_cond = pp[ip - sz_p] / sep
+                     - (z_mult_dat[ip - sz_p] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip - sz_p] * gravity *
+                     z_dir_g;
+
+        upper_cond = pp[ip] / sep + (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip] * gravity *
+                     z_dir_g;
+
+        diff = (lower_cond - upper_cond);
+
+        u_old = h
+                * PMeanDZ(permzp[ip - sz_p], permzp[ip],
+                          z_mult_dat[ip - sz_p], z_mult_dat[ip])
+                * diff
+                * RPMean(lower_cond, upper_cond,
+                         rpp[ip - sz_p] * dp[ip - sz_p], rpp[ip] * dp[ip])
+                / viscosity;
+
+        sep = dz * z_mult_dat[ip];
+
+        lower_cond = value / sep - 0.25 * dp[ip] * gravity;
+        upper_cond = pp[ip] / sep + 0.25 * dp[ip] * gravity;
+        diff = (lower_cond - upper_cond);
+        u_new = RPMean(lower_cond, upper_cond,
+                       rpp[ip - sz_p] * dp[ip - sz_p], rpp[ip] * dp[ip]);
+        u_new = u_new * h *
+                (permzp[ip] / viscosity)
+                * 2.0 * diff;
+      }),
                            FACE(FrontFace,
-                           {
-                             h = ffz * del_x_slope * del_y_slope;
-                             vel_vec = vz;
-                             vel_idx = vz_l + sz_v;
+      {
+        h = ffz * del_x_slope * del_y_slope;
+        vel_vec = vz;
+        vel_idx = vz_l + sz_v;
 
-                             dir = 1;
+        dir = 1;
 
-                             /* Calculate upper face velocity.
-                              * @RMM added cos to g term to test terrain-following grid
-                              */
+        /* Calculate upper face velocity.
+         * @RMM added cos to g term to test terrain-following grid
+         */
 
-                             sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_p]); //RMM
+        sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_p]);                      //RMM
 
-                             lower_cond = pp[ip] / sep - (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p])) * dp[ip] * gravity *
-                                          z_dir_g;
+        lower_cond = pp[ip] / sep - (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p])) * dp[ip] * gravity *
+                     z_dir_g;
 
-                             upper_cond = pp[ip + sz_p] / sep
-                                          + (z_mult_dat[ip + sz_p] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p]))
-                                          * dp[ip + sz_p] * gravity * z_dir_g;
+        upper_cond = pp[ip + sz_p] / sep
+                     + (z_mult_dat[ip + sz_p] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p]))
+                     * dp[ip + sz_p] * gravity * z_dir_g;
 
-                             diff = (lower_cond - upper_cond);
+        diff = (lower_cond - upper_cond);
 
 
-                             u_old = h
-                                     * PMeanDZ(permzp[ip], permzp[ip + sz_p],
-                                               z_mult_dat[ip], z_mult_dat[ip + sz_p])
-                                     * diff
-                                     * RPMean(lower_cond, upper_cond,
-                                              rpp[ip] * dp[ip], rpp[ip + sz_p] * dp[ip + sz_p])
-                                     / viscosity;
+        u_old = h
+                * PMeanDZ(permzp[ip], permzp[ip + sz_p],
+                          z_mult_dat[ip], z_mult_dat[ip + sz_p])
+                * diff
+                * RPMean(lower_cond, upper_cond,
+                         rpp[ip] * dp[ip], rpp[ip + sz_p] * dp[ip + sz_p])
+                / viscosity;
 
-                             sep = dz * z_mult_dat[ip];
+        sep = dz * z_mult_dat[ip];
 
-                             lower_cond = (pp[ip] / sep) - 0.25 * dp[ip] * gravity * z_dir_g;
-                             upper_cond = (value / sep) + 0.25 * dp[ip] * gravity * z_dir_g;
+        lower_cond = (pp[ip] / sep) - 0.25 * dp[ip] * gravity * z_dir_g;
+        upper_cond = (value / sep) + 0.25 * dp[ip] * gravity * z_dir_g;
 
-                             diff = lower_cond - upper_cond;
-                             u_new = RPMean(lower_cond, upper_cond,
-                                            rpp[ip] * dp[ip], rpp[ip + sz_p] * dp[ip + sz_p]);
-                             u_new = u_new * h *
-                                     (permzp[ip] / viscosity)
-                                     * 2.0 * diff;
-                           }),
+        diff = lower_cond - upper_cond;
+        u_new = RPMean(lower_cond, upper_cond,
+                       rpp[ip] * dp[ip], rpp[ip + sz_p] * dp[ip + sz_p]);
+        u_new = u_new * h *
+                (permzp[ip] / viscosity)
+                * 2.0 * diff;
+      }),
                            CellFinalize(
-                           {
+      {
 /* Remove the boundary term computed above */
-                             fp[ip] -= dt * dir * u_old;
+        fp[ip] -= dt * dir * u_old;
 
-                             /* Add the correct boundary term */
-                             fp[ip] += dt * dir * u_new;
+        /* Add the correct boundary term */
+        fp[ip] += dt * dir * u_new;
 
-                             vel_vec[vel_idx] = u_new / h;
-                           }),
+        vel_vec[vel_idx] = u_new / h;
+      }),
                            AfterAllCells(DoNothing)
-        ); /* End DirichletBC */
+                           ); /* End DirichletBC */
 
       ForPatchCellsPerFace(FluxBC,
                            BeforeAllCells(DoNothing),
@@ -1145,226 +1144,226 @@ void NlFunctionEval(Vector *     pressure, /* Current pressure values */
                                   double diff, u_new, u_old, h;
                                   double x_dir_g, y_dir_g, z_dir_g;
                                   double sep, lower_cond, upper_cond;
-                                  double del_x_slope, del_y_slope;),
+                                  double del_x_slope, del_y_slope; ),
                            CellSetup(
-                           {
-                             dir = 0;
-                             ip = SubvectorEltIndex(p_sub, i, j, k);
-
-                             vel_idx = 0;
-                             vx_l = SubvectorEltIndex(vx_sub, i, j, k);
-                             vy_l = SubvectorEltIndex(vy_sub, i, j, k);
-                             vz_l = SubvectorEltIndex(vz_sub, i, j, k);
-                             vel_vec = NULL;
-                             h = 0.0;
-
-                             diff = 0.0e0;
-                             u_new = 0.0e0;
-                             u_old = 0.0e0;
-
-                             x_dir_g = 0.0;
-                             y_dir_g = 0.0;
-                             z_dir_g = 1.0;
-
-                             sep = 0.0;
-                             lower_cond = 0.0;
-                             upper_cond = 0.0;
-
-                             del_x_slope = 1.0;
-                             del_y_slope = 1.0;
-                           }),
+      {
+        dir = 0;
+        ip = SubvectorEltIndex(p_sub, i, j, k);
+
+        vel_idx = 0;
+        vx_l = SubvectorEltIndex(vx_sub, i, j, k);
+        vy_l = SubvectorEltIndex(vy_sub, i, j, k);
+        vz_l = SubvectorEltIndex(vz_sub, i, j, k);
+        vel_vec = NULL;
+        h = 0.0;
+
+        diff = 0.0e0;
+        u_new = 0.0e0;
+        u_old = 0.0e0;
+
+        x_dir_g = 0.0;
+        y_dir_g = 0.0;
+        z_dir_g = 1.0;
+
+        sep = 0.0;
+        lower_cond = 0.0;
+        upper_cond = 0.0;
+
+        del_x_slope = 1.0;
+        del_y_slope = 1.0;
+      }),
                            FACE(LeftFace,
-                           {
-                             h = ffx * z_mult_dat[ip] * del_y_slope;
-                             vel_vec = vx;
-                             vel_idx = vx_l;
-
-                             dir = -1;
-                             diff = pp[ip - 1] - pp[ip];
-                             u_old = h
-                                     * PMean(pp[ip - 1], pp[ip],
-                                             permxp[ip - 1], permxp[ip])
-                                     * (diff / dx * del_x_slope)
-                                     * RPMean(pp[ip - 1], pp[ip],
-                                              rpp[ip - 1] * dp[ip - 1], rpp[ip] * dp[ip])
-                                     / viscosity;
-
-                             u_old += h
-                                      * PMean(pp[ip - 1], pp[ip],
-                                              permxp[ip - 1], permxp[ip])
-                                      * (-x_dir_g)
-                                      * RPMean(pp[ip - 1], pp[ip], rpp[ip - 1] * dp[ip - 1],
-                                               rpp[ip] * dp[ip])
-                                      / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffx * z_mult_dat[ip] * del_y_slope;
+        vel_vec = vx;
+        vel_idx = vx_l;
+
+        dir = -1;
+        diff = pp[ip - 1] - pp[ip];
+        u_old = h
+                * PMean(pp[ip - 1], pp[ip],
+                        permxp[ip - 1], permxp[ip])
+                * (diff / dx * del_x_slope)
+                * RPMean(pp[ip - 1], pp[ip],
+                         rpp[ip - 1] * dp[ip - 1], rpp[ip] * dp[ip])
+                / viscosity;
+
+        u_old += h
+                 * PMean(pp[ip - 1], pp[ip],
+                         permxp[ip - 1], permxp[ip])
+                 * (-x_dir_g)
+                 * RPMean(pp[ip - 1], pp[ip], rpp[ip - 1] * dp[ip - 1],
+                          rpp[ip] * dp[ip])
+                 / viscosity;
+        u_new = h;
+      }),
                            FACE(RightFace,
-                           {
-                             h = ffx * z_mult_dat[ip] * del_y_slope;
-                             vel_vec = vx;
-                             vel_idx = vx_l + 1;
-
-                             dir = 1;
-                             diff = pp[ip] - pp[ip + 1];
-                             u_old = h
-                                     * PMean(pp[ip], pp[ip + 1],
-                                             permxp[ip], permxp[ip + 1])
-                                     * (diff / dx * del_x_slope)
-                                     * RPMean(pp[ip], pp[ip + 1],
-                                              rpp[ip] * dp[ip], rpp[ip + 1] * dp[ip + 1])
-                                     / viscosity;
-
-                             u_old += h
-                                      * PMean(pp[ip], pp[ip + 1],
-                                              permxp[ip], permxp[ip + 1])
-                                      * (-x_dir_g)
-                                      * RPMean(pp[ip], pp[ip + 1], rpp[ip] * dp[ip],
-                                               rpp[ip + 1] * dp[ip + 1])
-                                      / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffx * z_mult_dat[ip] * del_y_slope;
+        vel_vec = vx;
+        vel_idx = vx_l + 1;
+
+        dir = 1;
+        diff = pp[ip] - pp[ip + 1];
+        u_old = h
+                * PMean(pp[ip], pp[ip + 1],
+                        permxp[ip], permxp[ip + 1])
+                * (diff / dx * del_x_slope)
+                * RPMean(pp[ip], pp[ip + 1],
+                         rpp[ip] * dp[ip], rpp[ip + 1] * dp[ip + 1])
+                / viscosity;
+
+        u_old += h
+                 * PMean(pp[ip], pp[ip + 1],
+                         permxp[ip], permxp[ip + 1])
+                 * (-x_dir_g)
+                 * RPMean(pp[ip], pp[ip + 1], rpp[ip] * dp[ip],
+                          rpp[ip + 1] * dp[ip + 1])
+                 / viscosity;
+        u_new = h;
+      }),
                            FACE(DownFace,
-                           {
-                             h = ffy * z_mult_dat[ip] * del_x_slope;
-                             vel_vec = vy;
-                             vel_idx = vy_l;
-
-                             dir = -1;
-                             diff = pp[ip - sy_p] - pp[ip];
-                             u_old = h
-                                     * PMean(pp[ip - sy_p], pp[ip],
-                                             permyp[ip - sy_p], permyp[ip])
-                                     * (diff / dy)
-                                     * RPMean(pp[ip - sy_p], pp[ip],
-                                              rpp[ip - sy_p] * dp[ip - sy_p], rpp[ip] * dp[ip])
-                                     / viscosity;
-
-                             u_old += h *
-                                      PMean(pp[ip], pp[ip - sy_p], permyp[ip],
-                                            permyp[ip - sy_p])
-                                      * (-y_dir_g)
-                                      * RPMean(pp[ip], pp[ip - sy_p], rpp[ip] * dp[ip],
-                                               rpp[ip - sy_p] * dp[ip - sy_p])
-                                      / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffy * z_mult_dat[ip] * del_x_slope;
+        vel_vec = vy;
+        vel_idx = vy_l;
+
+        dir = -1;
+        diff = pp[ip - sy_p] - pp[ip];
+        u_old = h
+                * PMean(pp[ip - sy_p], pp[ip],
+                        permyp[ip - sy_p], permyp[ip])
+                * (diff / dy)
+                * RPMean(pp[ip - sy_p], pp[ip],
+                         rpp[ip - sy_p] * dp[ip - sy_p], rpp[ip] * dp[ip])
+                / viscosity;
+
+        u_old += h *
+                 PMean(pp[ip], pp[ip - sy_p], permyp[ip],
+                       permyp[ip - sy_p])
+                 * (-y_dir_g)
+                 * RPMean(pp[ip], pp[ip - sy_p], rpp[ip] * dp[ip],
+                          rpp[ip - sy_p] * dp[ip - sy_p])
+                 / viscosity;
+        u_new = h;
+      }),
                            FACE(UpFace,
-                           {
-                             h = ffy * z_mult_dat[ip] * del_x_slope;
-                             vel_vec = vy;
-                             vel_idx = vy_l + sy_v;
-
-                             dir = 1;
-                             diff = pp[ip] - pp[ip + sy_p];
-                             u_old = h
-                                     * PMean(pp[ip], pp[ip + sy_p],
-                                             permyp[ip], permyp[ip + sy_p])
-                                     * (diff / dy)
-                                     * RPMean(pp[ip], pp[ip + sy_p],
-                                              rpp[ip] * dp[ip], rpp[ip + sy_p] * dp[ip + sy_p])
-                                     / viscosity;
-
-                             u_old += h
-                                      * PMean(pp[ip], pp[ip + sy_p], permyp[ip],
-                                              permyp[ip + sy_p])
-                                      * (-y_dir_g)
-                                      * RPMean(pp[ip], pp[ip + sy_p], rpp[ip] * dp[ip],
-                                               rpp[ip + sy_p] * dp[ip + sy_p])
-                                      / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffy * z_mult_dat[ip] * del_x_slope;
+        vel_vec = vy;
+        vel_idx = vy_l + sy_v;
+
+        dir = 1;
+        diff = pp[ip] - pp[ip + sy_p];
+        u_old = h
+                * PMean(pp[ip], pp[ip + sy_p],
+                        permyp[ip], permyp[ip + sy_p])
+                * (diff / dy)
+                * RPMean(pp[ip], pp[ip + sy_p],
+                         rpp[ip] * dp[ip], rpp[ip + sy_p] * dp[ip + sy_p])
+                / viscosity;
+
+        u_old += h
+                 * PMean(pp[ip], pp[ip + sy_p], permyp[ip],
+                         permyp[ip + sy_p])
+                 * (-y_dir_g)
+                 * RPMean(pp[ip], pp[ip + sy_p], rpp[ip] * dp[ip],
+                          rpp[ip + sy_p] * dp[ip + sy_p])
+                 / viscosity;
+        u_new = h;
+      }),
                            FACE(BackFace,
-                           {
-                             h = ffz * del_x_slope * del_y_slope;
-                             vel_vec = vz;
-                             vel_idx = vz_l;
-
-                             dir = -1;
-                             sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_p]); //RMM
-
-                             lower_cond = (pp[ip - sz_p] / sep)
-                                          - (z_mult_dat[ip - sz_p] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip - sz_p] * gravity *
-                                          z_dir_g;
-
-                             upper_cond = (pp[ip] / sep) + (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip] * gravity *
-                                          z_dir_g;
-
-                             diff = lower_cond - upper_cond;
-                             u_old = h
-                                     * PMeanDZ(permzp[ip - sz_p], permzp[ip],
-                                               z_mult_dat[ip - sz_p], z_mult_dat[ip])
-                                     * diff
-                                     * RPMean(lower_cond, upper_cond,
-                                              rpp[ip - sz_p] * dp[ip - sz_p], rpp[ip] * dp[ip])
-                                     / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffz * del_x_slope * del_y_slope;
+        vel_vec = vz;
+        vel_idx = vz_l;
+
+        dir = -1;
+        sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_p]);                      //RMM
+
+        lower_cond = (pp[ip - sz_p] / sep)
+                     - (z_mult_dat[ip - sz_p] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip - sz_p] * gravity *
+                     z_dir_g;
+
+        upper_cond = (pp[ip] / sep) + (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip] * gravity *
+                     z_dir_g;
+
+        diff = lower_cond - upper_cond;
+        u_old = h
+                * PMeanDZ(permzp[ip - sz_p], permzp[ip],
+                          z_mult_dat[ip - sz_p], z_mult_dat[ip])
+                * diff
+                * RPMean(lower_cond, upper_cond,
+                         rpp[ip - sz_p] * dp[ip - sz_p], rpp[ip] * dp[ip])
+                / viscosity;
+        u_new = h;
+      }),
                            FACE(FrontFace,
-                           {
-                             h = ffz * del_x_slope * del_y_slope;
-                             vel_vec = vz;
-                             vel_idx = vz_l + sz_v;
-
-                             dir = 1;
-                             sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_p]); //RMM
-
-                             lower_cond = (pp[ip] / sep) - (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p])) * dp[ip] * gravity *
-                                          z_dir_g;
-
-                             upper_cond = (pp[ip + sz_p] / sep)
-                                          + (z_mult_dat[ip + sz_p] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p])) * dp[ip + sz_p] * gravity *
-                                          z_dir_g;
-
-                             diff = lower_cond - upper_cond;
-                             u_old = h
-                                     * PMeanDZ(permzp[ip], permzp[ip + sz_p],
-                                               z_mult_dat[ip], z_mult_dat[ip + sz_p])
-                                     * diff
-                                     * RPMean(lower_cond, upper_cond,
-                                              rpp[ip] * dp[ip], rpp[ip + sz_p] * dp[ip + sz_p])
-                                     / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffz * del_x_slope * del_y_slope;
+        vel_vec = vz;
+        vel_idx = vz_l + sz_v;
+
+        dir = 1;
+        sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_p]);                      //RMM
+
+        lower_cond = (pp[ip] / sep) - (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p])) * dp[ip] * gravity *
+                     z_dir_g;
+
+        upper_cond = (pp[ip + sz_p] / sep)
+                     + (z_mult_dat[ip + sz_p] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p])) * dp[ip + sz_p] * gravity *
+                     z_dir_g;
+
+        diff = lower_cond - upper_cond;
+        u_old = h
+                * PMeanDZ(permzp[ip], permzp[ip + sz_p],
+                          z_mult_dat[ip], z_mult_dat[ip + sz_p])
+                * diff
+                * RPMean(lower_cond, upper_cond,
+                         rpp[ip] * dp[ip], rpp[ip + sz_p] * dp[ip + sz_p])
+                / viscosity;
+        u_new = h;
+      }),
                            CellFinalize(
-                           {
-                             /* Remove the boundary term computed above */
-                             fp[ip] -= dt * dir * u_old;
-                             /* Add the correct boundary term */
-                             u_new = u_new * bc_patch_values[ival];
-                             fp[ip] += dt * dir * u_new;
-                             
-                             vel_vec[vel_idx] = bc_patch_values[ival];
-                           }),
+      {
+        /* Remove the boundary term computed above */
+        fp[ip] -= dt * dir * u_old;
+        /* Add the correct boundary term */
+        u_new = u_new * bc_patch_values[ival];
+        fp[ip] += dt * dir * u_new;
+
+        vel_vec[vel_idx] = bc_patch_values[ival];
+      }),
                            AfterAllCells(DoNothing)
-        ); /* End FluxBC */
+                           ); /* End FluxBC */
 
       ForPatchCellsPerFace(OverlandBC,
                            BeforeAllCells(
-                           {
-                             if (diffusive == 0)
-                             {
-                               /* Call overlandflow_eval to compute fluxes across the east, west, north, and south faces */
-                               PFModuleInvokeType(OverlandFlowEvalInvoke, overlandflow_module,
-                                                  (grid, is, bc_struct, ipatch,
-                                                   problem_data, pressure, old_pressure,
-                                                   ke_, kw_, kn_, ks_, qx_, qy_, CALCFCN));
-                             }
-                             else
-                             {
-                               /*  @RMM this is modified to be kinematic wave routing, with a new module for diffusive wave
-                                * routing added */
-                               double *dummy1 = NULL;
-                               double *dummy2 = NULL;
-                               double *dummy3 = NULL;
-                               double *dummy4 = NULL;
-                               PFModuleInvokeType(OverlandFlowEvalDiffInvoke, overlandflow_module_diff,
-                                                  (grid, is, bc_struct, ipatch,
-                                                   problem_data, pressure, old_pressure,
-                                                   ke_, kw_, kn_, ks_,
-                                                   dummy1, dummy2, dummy3, dummy4,
-                                                   qx_, qy_, CALCFCN));
-                             }
-                           }),
+      {
+        if (diffusive == 0)
+        {
+          /* Call overlandflow_eval to compute fluxes across the east, west, north, and south faces */
+          PFModuleInvokeType(OverlandFlowEvalInvoke, overlandflow_module,
+                             (grid, is, bc_struct, ipatch,
+                              problem_data, pressure, old_pressure,
+                              ke_, kw_, kn_, ks_, qx_, qy_, CALCFCN));
+        }
+        else
+        {
+          /*  @RMM this is modified to be kinematic wave routing, with a new module for diffusive wave
+           * routing added */
+          double *dummy1 = NULL;
+          double *dummy2 = NULL;
+          double *dummy3 = NULL;
+          double *dummy4 = NULL;
+          PFModuleInvokeType(OverlandFlowEvalDiffInvoke, overlandflow_module_diff,
+                             (grid, is, bc_struct, ipatch,
+                              problem_data, pressure, old_pressure,
+                              ke_, kw_, kn_, ks_,
+                              dummy1, dummy2, dummy3, dummy4,
+                              qx_, qy_, CALCFCN));
+        }
+      }),
                            LoopVars(i, j, k, ival, bc_struct, ipatch, is),
                            Locals(int ip, io, dir;
                                   int vel_idx, vx_l, vy_l, vz_l;
@@ -1372,220 +1371,220 @@ void NlFunctionEval(Vector *     pressure, /* Current pressure values */
                                   double q_overlnd, u_old, u_new, diff, h;
                                   double x_dir_g, y_dir_g, z_dir_g;
                                   double sep, lower_cond, upper_cond;
-                                  double del_x_slope, del_y_slope;),
+                                  double del_x_slope, del_y_slope; ),
                            CellSetup(
-                           {
-                             dir = 0;
-                             ip = SubvectorEltIndex(p_sub, i, j, k);
-                             io = SubvectorEltIndex(x_sl_sub, i, j, 0);
-
-                             vel_idx = 0;
-                             vx_l = SubvectorEltIndex(vx_sub, i, j, k);
-                             vy_l = SubvectorEltIndex(vy_sub, i, j, k);
-                             vz_l = SubvectorEltIndex(vz_sub, i, j, k);
-                             vel_vec = NULL;
-                             h = 0.0;
-
-                             diff = 0.0e0;
-                             u_new = 0.0e0;
-                             u_old = 0.0e0;
-
-                             x_dir_g = 0.0;
-                             y_dir_g = 0.0;
-                             z_dir_g = 1.0;
-
-                             sep = 0.0;
-                             lower_cond = 0.0;
-                             upper_cond = 0.0;
-
-                             del_x_slope = 1.0;
-                             del_y_slope = 1.0;
-
-                             q_overlnd = 0;
-                           }),
+      {
+        dir = 0;
+        ip = SubvectorEltIndex(p_sub, i, j, k);
+        io = SubvectorEltIndex(x_sl_sub, i, j, 0);
+
+        vel_idx = 0;
+        vx_l = SubvectorEltIndex(vx_sub, i, j, k);
+        vy_l = SubvectorEltIndex(vy_sub, i, j, k);
+        vz_l = SubvectorEltIndex(vz_sub, i, j, k);
+        vel_vec = NULL;
+        h = 0.0;
+
+        diff = 0.0e0;
+        u_new = 0.0e0;
+        u_old = 0.0e0;
+
+        x_dir_g = 0.0;
+        y_dir_g = 0.0;
+        z_dir_g = 1.0;
+
+        sep = 0.0;
+        lower_cond = 0.0;
+        upper_cond = 0.0;
+
+        del_x_slope = 1.0;
+        del_y_slope = 1.0;
+
+        q_overlnd = 0;
+      }),
                            FACE(LeftFace,
-                           {
-                             h = ffx * z_mult_dat[ip] * del_y_slope;
-                             vel_vec = vx;
-                             vel_idx = vx_l;
-
-                             dir = -1;
-                             diff = pp[ip - 1] - pp[ip];
-                             u_old = h
-                                     * PMean(pp[ip - 1], pp[ip],
-                                             permxp[ip - 1], permxp[ip])
-                                     * (diff / dx)
-                                     * RPMean(pp[ip - 1], pp[ip],
-                                              rpp[ip - 1] * dp[ip - 1], rpp[ip] * dp[ip])
-                                     / viscosity;
-
-                             u_old += h
-                                      * PMean(pp[ip - 1], pp[ip],
-                                              permxp[ip - 1], permxp[ip])
-                                      * (-x_dir_g)
-                                      * RPMean(pp[ip - 1], pp[ip], rpp[ip - 1] * dp[ip - 1],
-                                               rpp[ip] * dp[ip])
-                                      / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffx * z_mult_dat[ip] * del_y_slope;
+        vel_vec = vx;
+        vel_idx = vx_l;
+
+        dir = -1;
+        diff = pp[ip - 1] - pp[ip];
+        u_old = h
+                * PMean(pp[ip - 1], pp[ip],
+                        permxp[ip - 1], permxp[ip])
+                * (diff / dx)
+                * RPMean(pp[ip - 1], pp[ip],
+                         rpp[ip - 1] * dp[ip - 1], rpp[ip] * dp[ip])
+                / viscosity;
+
+        u_old += h
+                 * PMean(pp[ip - 1], pp[ip],
+                         permxp[ip - 1], permxp[ip])
+                 * (-x_dir_g)
+                 * RPMean(pp[ip - 1], pp[ip], rpp[ip - 1] * dp[ip - 1],
+                          rpp[ip] * dp[ip])
+                 / viscosity;
+        u_new = h;
+      }),
                            FACE(RightFace,
-                           {
-                             h = ffx * z_mult_dat[ip] * del_y_slope;
-                             vel_vec = vx;
-                             vel_idx = vx_l + 1;
-
-                             dir = 1;
-                             diff = pp[ip] - pp[ip + 1];
-                             u_old = h
-                                     * PMean(pp[ip], pp[ip + 1],
-                                             permxp[ip], permxp[ip + 1])
-                                     * (diff / dx)
-                                     * RPMean(pp[ip], pp[ip + 1],
-                                              rpp[ip] * dp[ip], rpp[ip + 1] * dp[ip + 1])
-                                     / viscosity;
-
-                             u_old += h
-                                      * PMean(pp[ip], pp[ip + 1],
-                                              permxp[ip], permxp[ip + 1])
-                                      * (-x_dir_g)
-                                      * RPMean(pp[ip], pp[ip + 1], rpp[ip] * dp[ip],
-                                               rpp[ip + 1] * dp[ip + 1])
-                                      / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffx * z_mult_dat[ip] * del_y_slope;
+        vel_vec = vx;
+        vel_idx = vx_l + 1;
+
+        dir = 1;
+        diff = pp[ip] - pp[ip + 1];
+        u_old = h
+                * PMean(pp[ip], pp[ip + 1],
+                        permxp[ip], permxp[ip + 1])
+                * (diff / dx)
+                * RPMean(pp[ip], pp[ip + 1],
+                         rpp[ip] * dp[ip], rpp[ip + 1] * dp[ip + 1])
+                / viscosity;
+
+        u_old += h
+                 * PMean(pp[ip], pp[ip + 1],
+                         permxp[ip], permxp[ip + 1])
+                 * (-x_dir_g)
+                 * RPMean(pp[ip], pp[ip + 1], rpp[ip] * dp[ip],
+                          rpp[ip + 1] * dp[ip + 1])
+                 / viscosity;
+        u_new = h;
+      }),
                            FACE(DownFace,
-                           {
-                             h = ffy * z_mult_dat[ip] * del_x_slope;
-                             vel_vec = vy;
-                             vel_idx = vy_l;
-
-                             dir = -1;
-                             diff = pp[ip - sy_p] - pp[ip];
-                             u_old = h
-                                     * PMean(pp[ip - sy_p], pp[ip],
-                                             permyp[ip - sy_p], permyp[ip])
-                                     * (diff / dy)
-                                     * RPMean(pp[ip - sy_p], pp[ip],
-                                              rpp[ip - sy_p] * dp[ip - sy_p], rpp[ip] * dp[ip])
-                                     / viscosity;
-
-                             u_old += h *
-                                      PMean(pp[ip], pp[ip - sy_p], permyp[ip],
-                                            permyp[ip - sy_p])
-                                      * (-y_dir_g)
-                                      * RPMean(pp[ip], pp[ip - sy_p], rpp[ip] * dp[ip],
-                                               rpp[ip - sy_p] * dp[ip - sy_p])
-                                      / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffy * z_mult_dat[ip] * del_x_slope;
+        vel_vec = vy;
+        vel_idx = vy_l;
+
+        dir = -1;
+        diff = pp[ip - sy_p] - pp[ip];
+        u_old = h
+                * PMean(pp[ip - sy_p], pp[ip],
+                        permyp[ip - sy_p], permyp[ip])
+                * (diff / dy)
+                * RPMean(pp[ip - sy_p], pp[ip],
+                         rpp[ip - sy_p] * dp[ip - sy_p], rpp[ip] * dp[ip])
+                / viscosity;
+
+        u_old += h *
+                 PMean(pp[ip], pp[ip - sy_p], permyp[ip],
+                       permyp[ip - sy_p])
+                 * (-y_dir_g)
+                 * RPMean(pp[ip], pp[ip - sy_p], rpp[ip] * dp[ip],
+                          rpp[ip - sy_p] * dp[ip - sy_p])
+                 / viscosity;
+        u_new = h;
+      }),
                            FACE(UpFace,
-                           {
-                             h = ffy * z_mult_dat[ip] * del_x_slope;
-                             vel_vec = vy;
-                             vel_idx = vy_l + sy_v;
-
-                             dir = 1;
-                             diff = pp[ip] - pp[ip + sy_p];
-                             u_old = h
-                                     * PMean(pp[ip], pp[ip + sy_p],
-                                             permyp[ip], permyp[ip + sy_p])
-                                     * (diff / dy)
-                                     * RPMean(pp[ip], pp[ip + sy_p],
-                                              rpp[ip] * dp[ip], rpp[ip + sy_p] * dp[ip + sy_p])
-                                     / viscosity;
-
-                             u_old += h
-                                      * PMean(pp[ip], pp[ip + sy_p], permyp[ip],
-                                              permyp[ip + sy_p])
-                                      * (-y_dir_g)
-                                      * RPMean(pp[ip], pp[ip + sy_p], rpp[ip] * dp[ip],
-                                               rpp[ip + sy_p] * dp[ip + sy_p])
-                                      / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffy * z_mult_dat[ip] * del_x_slope;
+        vel_vec = vy;
+        vel_idx = vy_l + sy_v;
+
+        dir = 1;
+        diff = pp[ip] - pp[ip + sy_p];
+        u_old = h
+                * PMean(pp[ip], pp[ip + sy_p],
+                        permyp[ip], permyp[ip + sy_p])
+                * (diff / dy)
+                * RPMean(pp[ip], pp[ip + sy_p],
+                         rpp[ip] * dp[ip], rpp[ip + sy_p] * dp[ip + sy_p])
+                / viscosity;
+
+        u_old += h
+                 * PMean(pp[ip], pp[ip + sy_p], permyp[ip],
+                         permyp[ip + sy_p])
+                 * (-y_dir_g)
+                 * RPMean(pp[ip], pp[ip + sy_p], rpp[ip] * dp[ip],
+                          rpp[ip + sy_p] * dp[ip + sy_p])
+                 / viscosity;
+        u_new = h;
+      }),
                            FACE(BackFace,
-                           {
-                             h = ffz * del_x_slope * del_y_slope;
-                             vel_vec = vz;
-                             vel_idx = vz_l;
-
-                             dir = -1;
-                             sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_p]); //RMM
-                             //  sep = dz*z_mult_dat[ip];  //RMM
-
-                             lower_cond = (pp[ip - sz_p] / sep)
-                                          - (z_mult_dat[ip - sz_p] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip - sz_p] * gravity *
-                                          z_dir_g;
-                             upper_cond = (pp[ip] / sep) + (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip] * gravity *
-                                          z_dir_g;
-
-                             diff = lower_cond - upper_cond;
-                             u_old = h
-                                     * PMeanDZ(permzp[ip - sz_p], permzp[ip],
-                                               z_mult_dat[ip - sz_p], z_mult_dat[ip])
-                                     * diff
-                                     * RPMean(lower_cond, upper_cond,
-                                              rpp[ip - sz_p] * dp[ip - sz_p], rpp[ip] * dp[ip])
-                                     / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffz * del_x_slope * del_y_slope;
+        vel_vec = vz;
+        vel_idx = vz_l;
+
+        dir = -1;
+        sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_p]);                      //RMM
+        //  sep = dz*z_mult_dat[ip];  //RMM
+
+        lower_cond = (pp[ip - sz_p] / sep)
+                     - (z_mult_dat[ip - sz_p] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip - sz_p] * gravity *
+                     z_dir_g;
+        upper_cond = (pp[ip] / sep) + (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip] * gravity *
+                     z_dir_g;
+
+        diff = lower_cond - upper_cond;
+        u_old = h
+                * PMeanDZ(permzp[ip - sz_p], permzp[ip],
+                          z_mult_dat[ip - sz_p], z_mult_dat[ip])
+                * diff
+                * RPMean(lower_cond, upper_cond,
+                         rpp[ip - sz_p] * dp[ip - sz_p], rpp[ip] * dp[ip])
+                / viscosity;
+        u_new = h;
+      }),
                            FACE(FrontFace,
-                           {
-                             h = ffz * del_x_slope * del_y_slope;
-                             vel_vec = vz;
-                             vel_idx = vz_l + sz_v;
-
-                             dir = 1;
-                             sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_p]); //RMM
-
-                             lower_cond = (pp[ip] / sep) - (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p])) * dp[ip] * gravity *
-                                          z_dir_g;
-                             upper_cond = (pp[ip + sz_p] / sep)
-                                          + (z_mult_dat[ip + sz_p] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p])) * dp[ip + sz_p] * gravity *
-                                          z_dir_g;
-                             diff = lower_cond - upper_cond;
-                             u_old = h
-                                     * PMeanDZ(permzp[ip], permzp[ip + sz_p],
-                                               z_mult_dat[ip], z_mult_dat[ip + sz_p])
-                                     * diff
-                                     * RPMean(lower_cond, upper_cond,
-                                              rpp[ip] * dp[ip], rpp[ip + sz_p] * dp[ip + sz_p])
-                                     / viscosity;
-                             u_new = h;
-
-                             /* Add overland contribs */
-                             q_overlnd = 0.0;
-                             q_overlnd = vol
-                                         * (pfmax(pp[ip], 0.0) - pfmax(opp[ip], 0.0)) / dz +
-                                         dt * vol * ((ke_[io] - kw_[io]) / dx + (kn_[io] - ks_[io]) / dy)
-                                         / dz + vol * dt / dz * (exp(pfmin(pp[ip], 0.0) * public_xtra->SpinupDampP1) * public_xtra->SpinupDampP2);
-                             //NBE
-
-                             if (overlandspinup == 1)
-                             {
-                               /* add flux loss equal to excess head  that overwrites the prior overland flux */
-                               q_overlnd = (vol / dz) * dt
-                                           * ((pfmax(pp[ip], 0.0) - 0.0)
-                                              + exp(pfmin(pp[ip], 0.0) * public_xtra->SpinupDampP1)
-                                              * public_xtra->SpinupDampP2); //@RMM
-                             }
-                             fp[ip] += q_overlnd;
-                           }),
+      {
+        h = ffz * del_x_slope * del_y_slope;
+        vel_vec = vz;
+        vel_idx = vz_l + sz_v;
+
+        dir = 1;
+        sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_p]);                      //RMM
+
+        lower_cond = (pp[ip] / sep) - (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p])) * dp[ip] * gravity *
+                     z_dir_g;
+        upper_cond = (pp[ip + sz_p] / sep)
+                     + (z_mult_dat[ip + sz_p] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p])) * dp[ip + sz_p] * gravity *
+                     z_dir_g;
+        diff = lower_cond - upper_cond;
+        u_old = h
+                * PMeanDZ(permzp[ip], permzp[ip + sz_p],
+                          z_mult_dat[ip], z_mult_dat[ip + sz_p])
+                * diff
+                * RPMean(lower_cond, upper_cond,
+                         rpp[ip] * dp[ip], rpp[ip + sz_p] * dp[ip + sz_p])
+                / viscosity;
+        u_new = h;
+
+        /* Add overland contribs */
+        q_overlnd = 0.0;
+        q_overlnd = vol
+                    * (pfmax(pp[ip], 0.0) - pfmax(opp[ip], 0.0)) / dz +
+                    dt * vol * ((ke_[io] - kw_[io]) / dx + (kn_[io] - ks_[io]) / dy)
+                    / dz + vol * dt / dz * (exp(pfmin(pp[ip], 0.0) * public_xtra->SpinupDampP1) * public_xtra->SpinupDampP2);
+        //NBE
+
+        if (overlandspinup == 1)
+        {
+          /* add flux loss equal to excess head  that overwrites the prior overland flux */
+          q_overlnd = (vol / dz) * dt
+                      * ((pfmax(pp[ip], 0.0) - 0.0)
+                         + exp(pfmin(pp[ip], 0.0) * public_xtra->SpinupDampP1)
+                         * public_xtra->SpinupDampP2);                      //@RMM
+        }
+        fp[ip] += q_overlnd;
+      }),
                            CellFinalize(
-                           {
+      {
 /* Remove the boundary term computed above */
-                             fp[ip] -= dt * dir * u_old;
-                             //add source boundary terms
-                             u_new = u_new * bc_patch_values[ival];       //sk: here we go in and implement surface routing!
+        fp[ip] -= dt * dir * u_old;
+        //add source boundary terms
+        u_new = u_new * bc_patch_values[ival];                            //sk: here we go in and implement surface routing!
 
-                             fp[ip] += dt * dir * u_new;
+        fp[ip] += dt * dir * u_new;
 
-                             u_new += q_overlnd/dt;
+        u_new += q_overlnd / dt;
 
-                             vel_vec[vel_idx] = u_new / h;
-                           }),
+        vel_vec[vel_idx] = u_new / h;
+      }),
                            AfterAllCells(DoNothing)
-        ); /* End OverlandBC case */
+                           ); /* End OverlandBC case */
 
       ForPatchCellsPerFace(SeepageFaceBC,
                            BeforeAllCells(DoNothing),
@@ -1594,108 +1593,108 @@ void NlFunctionEval(Vector *     pressure, /* Current pressure values */
                                   int vel_idx, vx_l, vy_l, vz_l;
                                   double *vel_vec;
                                   double u_new, u_old, q_overlnd, h;
-                                  double del_x_slope, del_y_slope;),
+                                  double del_x_slope, del_y_slope; ),
                            CellSetup(
-                           {
-                             dir = 0;
-                             ip = SubvectorEltIndex(p_sub, i, j, k);
-
-                             vel_idx = 0;
-                             vx_l = SubvectorEltIndex(vx_sub, i, j, k);
-                             vy_l = SubvectorEltIndex(vy_sub, i, j, k);
-                             vz_l = SubvectorEltIndex(vz_sub, i, j, k);
-                             vel_vec = NULL;
-                             h = 0.0;
-
-                             u_new = 0.0e0;
-                             u_old = 0.0e0;
-                             q_overlnd = 0.0;
-
-                             del_x_slope = 1.0;
-                             del_y_slope = 1.0;
-                           }),
+      {
+        dir = 0;
+        ip = SubvectorEltIndex(p_sub, i, j, k);
+
+        vel_idx = 0;
+        vx_l = SubvectorEltIndex(vx_sub, i, j, k);
+        vy_l = SubvectorEltIndex(vy_sub, i, j, k);
+        vz_l = SubvectorEltIndex(vz_sub, i, j, k);
+        vel_vec = NULL;
+        h = 0.0;
+
+        u_new = 0.0e0;
+        u_old = 0.0e0;
+        q_overlnd = 0.0;
+
+        del_x_slope = 1.0;
+        del_y_slope = 1.0;
+      }),
                            FACE(LeftFace, {
-                               h = ffx * z_mult_dat[ip] * del_y_slope;
-                               vel_vec = vx;
-                               vel_idx = vx_l;
+        h = ffx * z_mult_dat[ip] * del_y_slope;
+        vel_vec = vx;
+        vel_idx = vx_l;
 
-                               dir = -1;
-                               u_new = h;
-                             }),
+        dir = -1;
+        u_new = h;
+      }),
                            FACE(RightFace, {
-                               h = ffx * z_mult_dat[ip]* del_y_slope;
-                               vel_vec = vx;
-                               vel_idx = vx_l + 1;
+        h = ffx * z_mult_dat[ip] * del_y_slope;
+        vel_vec = vx;
+        vel_idx = vx_l + 1;
 
-                               dir = 1;
-                               u_new = h;
-                             }),
+        dir = 1;
+        u_new = h;
+      }),
                            FACE(DownFace, {
-                               h = ffy * z_mult_dat[ip] * del_x_slope;
-                               vel_vec = vy;
-                               vel_idx = vy_l;
+        h = ffy * z_mult_dat[ip] * del_x_slope;
+        vel_vec = vy;
+        vel_idx = vy_l;
 
-                               dir = -1;
-                               u_new = h;
-                             }),
+        dir = -1;
+        u_new = h;
+      }),
                            FACE(UpFace, {
-                               h = ffy * z_mult_dat[ip] * del_x_slope;
-                               vel_vec = vy;
-                               vel_idx = vy_l + sy_v;
+        h = ffy * z_mult_dat[ip] * del_x_slope;
+        vel_vec = vy;
+        vel_idx = vy_l + sy_v;
 
-                               dir = 1;
-                               u_new = h;
-                             }),
+        dir = 1;
+        u_new = h;
+      }),
                            FACE(BackFace, {
-                               h = ffz * del_x_slope * del_y_slope;
-                               vel_vec = vz;
-                               vel_idx = vz_l;
+        h = ffz * del_x_slope * del_y_slope;
+        vel_vec = vz;
+        vel_idx = vz_l;
 
-                               dir = -1;
-                               u_new = h;
-                             }),
+        dir = -1;
+        u_new = h;
+      }),
                            FACE(FrontFace, {
-                               h = ffz * del_x_slope * del_y_slope;
-                               vel_vec = vz;
-                               vel_idx = vz_l + sz_v;
-                               dir = 1;
-                               u_new = h;
+        h = ffz * del_x_slope * del_y_slope;
+        vel_vec = vz;
+        vel_idx = vz_l + sz_v;
+        dir = 1;
+        u_new = h;
 
 /* add flux loss equal to excess head that overwrites the prior overland flux */
-                               q_overlnd = (vol / dz) * dt * (pfmax(pp[ip], 0.0) - 0.0); //@RMM
+        q_overlnd = (vol / dz) * dt * (pfmax(pp[ip], 0.0) - 0.0);                        //@RMM
 
-                               fp[ip] += q_overlnd;
-                             }),
+        fp[ip] += q_overlnd;
+      }),
                            CellFinalize(
-                           {
+      {
 /* Remove the boundary condition computed above */
-                             fp[ip] -= dt * dir * u_old;
-                             // add source boundary terms
-                             u_new = u_new * bc_patch_values[ival];
-                             fp[ip] += dt * dir * u_new;
+        fp[ip] -= dt * dir * u_old;
+        // add source boundary terms
+        u_new = u_new * bc_patch_values[ival];
+        fp[ip] += dt * dir * u_new;
 
-                             u_new += q_overlnd/dt;
+        u_new += q_overlnd / dt;
 
-                             vel_vec[vel_idx] = u_new / h;
-                           }),
+        vel_vec[vel_idx] = u_new / h;
+      }),
                            AfterAllCells(DoNothing)
-        ); /* End SeepageFaceBC case */
+                           ); /* End SeepageFaceBC case */
 
       ForPatchCellsPerFace(OverlandKinematicBC,
                            BeforeAllCells(
-                           {
+      {
 /*  @RMM this is modified to be kinematic wave routing, with a new module for diffusive wave
  * routing added */
-                               double *dummy1 = NULL;
-                               double *dummy2 = NULL;
-                               double *dummy3 = NULL;
-                               double *dummy4 = NULL;
-                             PFModuleInvokeType(OverlandFlowEvalKinInvoke, overlandflow_module_kin,
-                                                (grid, is, bc_struct, ipatch, problem_data, pressure,
-                                                 ke_, kw_, kn_, ks_,
-                                                 dummy1, dummy2, dummy3, dummy4,
-                                                 qx_, qy_, CALCFCN));
-                           }),
+        double *dummy1 = NULL;
+        double *dummy2 = NULL;
+        double *dummy3 = NULL;
+        double *dummy4 = NULL;
+        PFModuleInvokeType(OverlandFlowEvalKinInvoke, overlandflow_module_kin,
+                           (grid, is, bc_struct, ipatch, problem_data, pressure,
+                            ke_, kw_, kn_, ks_,
+                            dummy1, dummy2, dummy3, dummy4,
+                            qx_, qy_, CALCFCN));
+      }),
                            LoopVars(i, j, k, ival, bc_struct, ipatch, is),
                            Locals(int ip, io, dir;
                                   int vel_idx, vx_l, vy_l, vz_l;
@@ -1703,226 +1702,226 @@ void NlFunctionEval(Vector *     pressure, /* Current pressure values */
                                   double q_overlnd, u_old, u_new, diff, h;
                                   double x_dir_g, y_dir_g, z_dir_g;
                                   double sep, lower_cond, upper_cond;
-                                  double del_x_slope, del_y_slope;),
+                                  double del_x_slope, del_y_slope; ),
                            CellSetup(
-                           {
-                             ip = SubvectorEltIndex(p_sub, i, j, k);
-                             io = SubvectorEltIndex(x_sl_sub, i, j, 0);
-                             dir = 0;
-
-                             vel_idx = 0;
-                             vx_l = SubvectorEltIndex(vx_sub, i, j, k);
-                             vy_l = SubvectorEltIndex(vy_sub, i, j, k);
-                             vz_l = SubvectorEltIndex(vz_sub, i, j, k);
-                             vel_vec = NULL;
-                             h = 0.0;
-
-                             diff = 0.0e0;
-                             u_new = 0.0e0;
-                             u_old = 0.0e0;
-
-                             x_dir_g = 0.0;
-                             y_dir_g = 0.0;
-                             z_dir_g = 1.0;
-
-                             sep = 0.0;
-                             lower_cond = 0.0;
-                             upper_cond = 0.0;
-
-                             del_x_slope = 1.0;
-                             del_y_slope = 1.0;
-
-                             q_overlnd = 0;
-                           }),
+      {
+        ip = SubvectorEltIndex(p_sub, i, j, k);
+        io = SubvectorEltIndex(x_sl_sub, i, j, 0);
+        dir = 0;
+
+        vel_idx = 0;
+        vx_l = SubvectorEltIndex(vx_sub, i, j, k);
+        vy_l = SubvectorEltIndex(vy_sub, i, j, k);
+        vz_l = SubvectorEltIndex(vz_sub, i, j, k);
+        vel_vec = NULL;
+        h = 0.0;
+
+        diff = 0.0e0;
+        u_new = 0.0e0;
+        u_old = 0.0e0;
+
+        x_dir_g = 0.0;
+        y_dir_g = 0.0;
+        z_dir_g = 1.0;
+
+        sep = 0.0;
+        lower_cond = 0.0;
+        upper_cond = 0.0;
+
+        del_x_slope = 1.0;
+        del_y_slope = 1.0;
+
+        q_overlnd = 0;
+      }),
                            FACE(LeftFace,
-                           {
-                             h = ffx * z_mult_dat[ip] * del_y_slope;
-                             vel_vec = vx;
-                             vel_idx = vx_l;
-
-                             dir = -1;
-                             diff = pp[ip - 1] - pp[ip];
-                             u_old = h
-                                     * PMean(pp[ip - 1], pp[ip],
-                                             permxp[ip - 1], permxp[ip])
-                                     * (diff / dx)
-                                     * RPMean(pp[ip - 1], pp[ip],
-                                              rpp[ip - 1] * dp[ip - 1], rpp[ip] * dp[ip])
-                                     / viscosity;
-
-                             u_old += h
-                                      * PMean(pp[ip - 1], pp[ip],
-                                              permxp[ip - 1], permxp[ip])
-                                      * (-x_dir_g)
-                                      * RPMean(pp[ip - 1], pp[ip], rpp[ip - 1] * dp[ip - 1],
-                                               rpp[ip] * dp[ip])
-                                      / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffx * z_mult_dat[ip] * del_y_slope;
+        vel_vec = vx;
+        vel_idx = vx_l;
+
+        dir = -1;
+        diff = pp[ip - 1] - pp[ip];
+        u_old = h
+                * PMean(pp[ip - 1], pp[ip],
+                        permxp[ip - 1], permxp[ip])
+                * (diff / dx)
+                * RPMean(pp[ip - 1], pp[ip],
+                         rpp[ip - 1] * dp[ip - 1], rpp[ip] * dp[ip])
+                / viscosity;
+
+        u_old += h
+                 * PMean(pp[ip - 1], pp[ip],
+                         permxp[ip - 1], permxp[ip])
+                 * (-x_dir_g)
+                 * RPMean(pp[ip - 1], pp[ip], rpp[ip - 1] * dp[ip - 1],
+                          rpp[ip] * dp[ip])
+                 / viscosity;
+        u_new = h;
+      }),
                            FACE(RightFace,
-                           {
-                             h = ffx * z_mult_dat[ip] * del_y_slope;
-                             vel_vec = vx;
-                             vel_idx = vx_l + 1;
-
-                             dir = 1;
-                             diff = pp[ip] - pp[ip + 1];
-                             u_old = h
-                                     * PMean(pp[ip], pp[ip + 1],
-                                             permxp[ip], permxp[ip + 1])
-                                     * (diff / dx)
-                                     * RPMean(pp[ip], pp[ip + 1],
-                                              rpp[ip] * dp[ip], rpp[ip + 1] * dp[ip + 1])
-                                     / viscosity;
-
-                             u_old += h
-                                      * PMean(pp[ip], pp[ip + 1],
-                                              permxp[ip], permxp[ip + 1])
-                                      * (-x_dir_g)
-                                      * RPMean(pp[ip], pp[ip + 1], rpp[ip] * dp[ip],
-                                               rpp[ip + 1] * dp[ip + 1])
-                                      / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffx * z_mult_dat[ip] * del_y_slope;
+        vel_vec = vx;
+        vel_idx = vx_l + 1;
+
+        dir = 1;
+        diff = pp[ip] - pp[ip + 1];
+        u_old = h
+                * PMean(pp[ip], pp[ip + 1],
+                        permxp[ip], permxp[ip + 1])
+                * (diff / dx)
+                * RPMean(pp[ip], pp[ip + 1],
+                         rpp[ip] * dp[ip], rpp[ip + 1] * dp[ip + 1])
+                / viscosity;
+
+        u_old += h
+                 * PMean(pp[ip], pp[ip + 1],
+                         permxp[ip], permxp[ip + 1])
+                 * (-x_dir_g)
+                 * RPMean(pp[ip], pp[ip + 1], rpp[ip] * dp[ip],
+                          rpp[ip + 1] * dp[ip + 1])
+                 / viscosity;
+        u_new = h;
+      }),
                            FACE(DownFace,
-                           {
-                             h = ffy * z_mult_dat[ip] * del_x_slope;
-                             vel_vec = vy;
-                             vel_idx = vy_l;
-
-                             dir = -1;
-                             diff = pp[ip - sy_p] - pp[ip];
-                             u_old = h
-                                     * PMean(pp[ip - sy_p], pp[ip],
-                                             permyp[ip - sy_p], permyp[ip])
-                                     * (diff / dy)
-                                     * RPMean(pp[ip - sy_p], pp[ip],
-                                              rpp[ip - sy_p] * dp[ip - sy_p], rpp[ip] * dp[ip])
-                                     / viscosity;
-
-                             u_old += h *
-                                      PMean(pp[ip], pp[ip - sy_p], permyp[ip],
-                                            permyp[ip - sy_p])
-                                      * (-y_dir_g)
-                                      * RPMean(pp[ip], pp[ip - sy_p], rpp[ip] * dp[ip],
-                                               rpp[ip - sy_p] * dp[ip - sy_p])
-                                      / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffy * z_mult_dat[ip] * del_x_slope;
+        vel_vec = vy;
+        vel_idx = vy_l;
+
+        dir = -1;
+        diff = pp[ip - sy_p] - pp[ip];
+        u_old = h
+                * PMean(pp[ip - sy_p], pp[ip],
+                        permyp[ip - sy_p], permyp[ip])
+                * (diff / dy)
+                * RPMean(pp[ip - sy_p], pp[ip],
+                         rpp[ip - sy_p] * dp[ip - sy_p], rpp[ip] * dp[ip])
+                / viscosity;
+
+        u_old += h *
+                 PMean(pp[ip], pp[ip - sy_p], permyp[ip],
+                       permyp[ip - sy_p])
+                 * (-y_dir_g)
+                 * RPMean(pp[ip], pp[ip - sy_p], rpp[ip] * dp[ip],
+                          rpp[ip - sy_p] * dp[ip - sy_p])
+                 / viscosity;
+        u_new = h;
+      }),
                            FACE(UpFace,
-                           {
-                             h = ffy * z_mult_dat[ip] * del_x_slope;
-                             vel_vec = vy;
-                             vel_idx = vy_l + sy_v;
-
-                             dir = 1;
-                             diff = pp[ip] - pp[ip + sy_p];
-                             u_old = h
-                                     * PMean(pp[ip], pp[ip + sy_p],
-                                             permyp[ip], permyp[ip + sy_p])
-                                     * (diff / dy)
-                                     * RPMean(pp[ip], pp[ip + sy_p],
-                                              rpp[ip] * dp[ip], rpp[ip + sy_p] * dp[ip + sy_p])
-                                     / viscosity;
-
-                             u_old += h
-                                      * PMean(pp[ip], pp[ip + sy_p], permyp[ip],
-                                              permyp[ip + sy_p])
-                                      * (-y_dir_g)
-                                      * RPMean(pp[ip], pp[ip + sy_p], rpp[ip] * dp[ip],
-                                               rpp[ip + sy_p] * dp[ip + sy_p])
-                                      / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffy * z_mult_dat[ip] * del_x_slope;
+        vel_vec = vy;
+        vel_idx = vy_l + sy_v;
+
+        dir = 1;
+        diff = pp[ip] - pp[ip + sy_p];
+        u_old = h
+                * PMean(pp[ip], pp[ip + sy_p],
+                        permyp[ip], permyp[ip + sy_p])
+                * (diff / dy)
+                * RPMean(pp[ip], pp[ip + sy_p],
+                         rpp[ip] * dp[ip], rpp[ip + sy_p] * dp[ip + sy_p])
+                / viscosity;
+
+        u_old += h
+                 * PMean(pp[ip], pp[ip + sy_p], permyp[ip],
+                         permyp[ip + sy_p])
+                 * (-y_dir_g)
+                 * RPMean(pp[ip], pp[ip + sy_p], rpp[ip] * dp[ip],
+                          rpp[ip + sy_p] * dp[ip + sy_p])
+                 / viscosity;
+        u_new = h;
+      }),
                            FACE(BackFace,
-                           {
-                             h = ffz * del_x_slope * del_y_slope;
-                             vel_vec = vz;
-                             vel_idx = vz_l;
-
-                             dir = -1;
-                             sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_p]); //RMM
-                             //  sep = dz*z_mult_dat[ip];  //RMM
-
-                             lower_cond = (pp[ip - sz_p] / sep)
-                                          - (z_mult_dat[ip - sz_p] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip - sz_p] * gravity *
-                                          z_dir_g;
-                             upper_cond = (pp[ip] / sep) + (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip] * gravity *
-                                          z_dir_g;
-
-                             diff = lower_cond - upper_cond;
-                             u_old = h
-                                     * PMeanDZ(permzp[ip - sz_p], permzp[ip],
-                                               z_mult_dat[ip - sz_p], z_mult_dat[ip])
-                                     * diff
-                                     * RPMean(lower_cond, upper_cond,
-                                              rpp[ip - sz_p] * dp[ip - sz_p], rpp[ip] * dp[ip])
-                                     / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffz * del_x_slope * del_y_slope;
+        vel_vec = vz;
+        vel_idx = vz_l;
+
+        dir = -1;
+        sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_p]);                      //RMM
+        //  sep = dz*z_mult_dat[ip];  //RMM
+
+        lower_cond = (pp[ip - sz_p] / sep)
+                     - (z_mult_dat[ip - sz_p] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip - sz_p] * gravity *
+                     z_dir_g;
+        upper_cond = (pp[ip] / sep) + (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip] * gravity *
+                     z_dir_g;
+
+        diff = lower_cond - upper_cond;
+        u_old = h
+                * PMeanDZ(permzp[ip - sz_p], permzp[ip],
+                          z_mult_dat[ip - sz_p], z_mult_dat[ip])
+                * diff
+                * RPMean(lower_cond, upper_cond,
+                         rpp[ip - sz_p] * dp[ip - sz_p], rpp[ip] * dp[ip])
+                / viscosity;
+        u_new = h;
+      }),
                            FACE(FrontFace,
-                           {
-                             h = ffz * del_x_slope * del_y_slope;
-                             vel_vec = vz;
-                             vel_idx = vz_l + sz_v;
-
-                             dir = 1;
-                             sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_p]); //RMM
-
-                             lower_cond = (pp[ip] / sep) - (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p])) * dp[ip] * gravity *
-                                          z_dir_g;
-                             upper_cond = (pp[ip + sz_p] / sep)
-                                          + (z_mult_dat[ip + sz_p] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p])) * dp[ip + sz_p] * gravity *
-                                          z_dir_g;
-                             diff = lower_cond - upper_cond;
-                             u_old = h
-                                     * PMeanDZ(permzp[ip], permzp[ip + sz_p],
-                                               z_mult_dat[ip], z_mult_dat[ip + sz_p])
-                                     * diff
-                                     * RPMean(lower_cond, upper_cond,
-                                              rpp[ip] * dp[ip], rpp[ip + sz_p] * dp[ip + sz_p])
-                                     / viscosity;
-                             u_new = h;
-
-                             q_overlnd = 0.0;
-                             q_overlnd = vol
-                                         * (pfmax(pp[ip], 0.0) - pfmax(opp[ip], 0.0)) / dz +
-                                         dt * vol * ((ke_[io] - kw_[io]) / dx + (kn_[io] - ks_[io]) / dy)
-                                         / dz;
-                             fp[ip] += q_overlnd;
-                           }),
+      {
+        h = ffz * del_x_slope * del_y_slope;
+        vel_vec = vz;
+        vel_idx = vz_l + sz_v;
+
+        dir = 1;
+        sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_p]);                      //RMM
+
+        lower_cond = (pp[ip] / sep) - (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p])) * dp[ip] * gravity *
+                     z_dir_g;
+        upper_cond = (pp[ip + sz_p] / sep)
+                     + (z_mult_dat[ip + sz_p] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p])) * dp[ip + sz_p] * gravity *
+                     z_dir_g;
+        diff = lower_cond - upper_cond;
+        u_old = h
+                * PMeanDZ(permzp[ip], permzp[ip + sz_p],
+                          z_mult_dat[ip], z_mult_dat[ip + sz_p])
+                * diff
+                * RPMean(lower_cond, upper_cond,
+                         rpp[ip] * dp[ip], rpp[ip + sz_p] * dp[ip + sz_p])
+                / viscosity;
+        u_new = h;
+
+        q_overlnd = 0.0;
+        q_overlnd = vol
+                    * (pfmax(pp[ip], 0.0) - pfmax(opp[ip], 0.0)) / dz +
+                    dt * vol * ((ke_[io] - kw_[io]) / dx + (kn_[io] - ks_[io]) / dy)
+                    / dz;
+        fp[ip] += q_overlnd;
+      }),
                            CellFinalize(
-                           {
+      {
 /* Remove the boundary term computed above */
-                             fp[ip] -= dt * dir * u_old;
-                             //add source boundary terms
-                             u_new = u_new * bc_patch_values[ival];
+        fp[ip] -= dt * dir * u_old;
+        //add source boundary terms
+        u_new = u_new * bc_patch_values[ival];
 
-                             fp[ip] += dt * dir * u_new;
+        fp[ip] += dt * dir * u_new;
 
-                             u_new += q_overlnd/dt;
+        u_new += q_overlnd / dt;
 
-                             vel_vec[vel_idx] = u_new / h;
-                           }),
+        vel_vec[vel_idx] = u_new / h;
+      }),
                            AfterAllCells(DoNothing)
-        );
+                           );
 
       ForPatchCellsPerFace(OverlandDiffusiveBC,
                            BeforeAllCells(
-                           {
-                             /*  @RMM this is a new module for diffusive wave
-                              */
-                               double *dummy1 = NULL;
-                               double *dummy2 = NULL;
-                               double *dummy3 = NULL;
-                               double *dummy4 = NULL;
-                             PFModuleInvokeType(OverlandFlowEvalDiffInvoke, overlandflow_module_diff,
-                                                (grid, is, bc_struct, ipatch,
-                                                 problem_data, pressure, old_pressure,
-                                                 ke_, kw_, kn_, ks_,
-                                                 dummy1, dummy2, dummy3, dummy4,
-                                                 qx_, qy_, CALCFCN));
-                           }),
+      {
+        /*  @RMM this is a new module for diffusive wave
+         */
+        double *dummy1 = NULL;
+        double *dummy2 = NULL;
+        double *dummy3 = NULL;
+        double *dummy4 = NULL;
+        PFModuleInvokeType(OverlandFlowEvalDiffInvoke, overlandflow_module_diff,
+                           (grid, is, bc_struct, ipatch,
+                            problem_data, pressure, old_pressure,
+                            ke_, kw_, kn_, ks_,
+                            dummy1, dummy2, dummy3, dummy4,
+                            qx_, qy_, CALCFCN));
+      }),
                            LoopVars(i, j, k, ival, bc_struct, ipatch, is),
                            Locals(int ip, io, dir;
                                   int vel_idx, vx_l, vy_l, vz_l;
@@ -1930,210 +1929,210 @@ void NlFunctionEval(Vector *     pressure, /* Current pressure values */
                                   double q_overlnd, u_old, u_new, diff, h;
                                   double x_dir_g, y_dir_g, z_dir_g;
                                   double sep, lower_cond, upper_cond;
-                                  double del_x_slope, del_y_slope;),
+                                  double del_x_slope, del_y_slope; ),
                            CellSetup(
-                           {
-                             ip = SubvectorEltIndex(p_sub, i, j, k);
-                             io = SubvectorEltIndex(x_sl_sub, i, j, 0);
-                             dir = 0;
-
-                             vel_idx = 0;
-                             vx_l = SubvectorEltIndex(vx_sub, i, j, k);
-                             vy_l = SubvectorEltIndex(vy_sub, i, j, k);
-                             vz_l = SubvectorEltIndex(vz_sub, i, j, k);
-                             vel_vec = NULL;
-                             h = 0.0;
-
-                             diff = 0.0e0;
-                             u_new = 0.0e0;
-                             u_old = 0.0e0;
-
-                             x_dir_g = 0.0;
-                             y_dir_g = 0.0;
-                             z_dir_g = 1.0;
-
-                             sep = 0.0;
-                             lower_cond = 0.0;
-                             upper_cond = 0.0;
-
-                             del_x_slope = 1.0;
-                             del_y_slope = 1.0;
-
-                             q_overlnd = 0;
-                           }),
+      {
+        ip = SubvectorEltIndex(p_sub, i, j, k);
+        io = SubvectorEltIndex(x_sl_sub, i, j, 0);
+        dir = 0;
+
+        vel_idx = 0;
+        vx_l = SubvectorEltIndex(vx_sub, i, j, k);
+        vy_l = SubvectorEltIndex(vy_sub, i, j, k);
+        vz_l = SubvectorEltIndex(vz_sub, i, j, k);
+        vel_vec = NULL;
+        h = 0.0;
+
+        diff = 0.0e0;
+        u_new = 0.0e0;
+        u_old = 0.0e0;
+
+        x_dir_g = 0.0;
+        y_dir_g = 0.0;
+        z_dir_g = 1.0;
+
+        sep = 0.0;
+        lower_cond = 0.0;
+        upper_cond = 0.0;
+
+        del_x_slope = 1.0;
+        del_y_slope = 1.0;
+
+        q_overlnd = 0;
+      }),
                            FACE(LeftFace,
-                           {
-                             h = ffx * z_mult_dat[ip] * del_y_slope;
-                             vel_vec = vx;
-                             vel_idx = vx_l;
-
-                             dir = -1;
-                             diff = pp[ip - 1] - pp[ip];
-                             u_old = h
-                                     * PMean(pp[ip - 1], pp[ip],
-                                             permxp[ip - 1], permxp[ip])
-                                     * (diff / dx)
-                                     * RPMean(pp[ip - 1], pp[ip],
-                                              rpp[ip - 1] * dp[ip - 1], rpp[ip] * dp[ip])
-                                     / viscosity;
-
-                             u_old += h
-                                      * PMean(pp[ip - 1], pp[ip],
-                                              permxp[ip - 1], permxp[ip])
-                                      * (-x_dir_g)
-                                      * RPMean(pp[ip - 1], pp[ip], rpp[ip - 1] * dp[ip - 1],
-                                               rpp[ip] * dp[ip])
-                                      / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffx * z_mult_dat[ip] * del_y_slope;
+        vel_vec = vx;
+        vel_idx = vx_l;
+
+        dir = -1;
+        diff = pp[ip - 1] - pp[ip];
+        u_old = h
+                * PMean(pp[ip - 1], pp[ip],
+                        permxp[ip - 1], permxp[ip])
+                * (diff / dx)
+                * RPMean(pp[ip - 1], pp[ip],
+                         rpp[ip - 1] * dp[ip - 1], rpp[ip] * dp[ip])
+                / viscosity;
+
+        u_old += h
+                 * PMean(pp[ip - 1], pp[ip],
+                         permxp[ip - 1], permxp[ip])
+                 * (-x_dir_g)
+                 * RPMean(pp[ip - 1], pp[ip], rpp[ip - 1] * dp[ip - 1],
+                          rpp[ip] * dp[ip])
+                 / viscosity;
+        u_new = h;
+      }),
                            FACE(RightFace,
-                           {
-                             h = ffx * z_mult_dat[ip] * del_y_slope;
-                             vel_vec = vx;
-                             vel_idx = vx_l + 1;
-
-                             dir = 1;
-                             diff = pp[ip] - pp[ip + 1];
-                             u_old = h
-                                     * PMean(pp[ip], pp[ip + 1],
-                                             permxp[ip], permxp[ip + 1])
-                                     * (diff / dx)
-                                     * RPMean(pp[ip], pp[ip + 1],
-                                              rpp[ip] * dp[ip], rpp[ip + 1] * dp[ip + 1])
-                                     / viscosity;
-
-                             u_old += h
-                                      * PMean(pp[ip], pp[ip + 1],
-                                              permxp[ip], permxp[ip + 1])
-                                      * (-x_dir_g)
-                                      * RPMean(pp[ip], pp[ip + 1], rpp[ip] * dp[ip],
-                                               rpp[ip + 1] * dp[ip + 1])
-                                      / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffx * z_mult_dat[ip] * del_y_slope;
+        vel_vec = vx;
+        vel_idx = vx_l + 1;
+
+        dir = 1;
+        diff = pp[ip] - pp[ip + 1];
+        u_old = h
+                * PMean(pp[ip], pp[ip + 1],
+                        permxp[ip], permxp[ip + 1])
+                * (diff / dx)
+                * RPMean(pp[ip], pp[ip + 1],
+                         rpp[ip] * dp[ip], rpp[ip + 1] * dp[ip + 1])
+                / viscosity;
+
+        u_old += h
+                 * PMean(pp[ip], pp[ip + 1],
+                         permxp[ip], permxp[ip + 1])
+                 * (-x_dir_g)
+                 * RPMean(pp[ip], pp[ip + 1], rpp[ip] * dp[ip],
+                          rpp[ip + 1] * dp[ip + 1])
+                 / viscosity;
+        u_new = h;
+      }),
                            FACE(DownFace,
-                           {
-                             h = ffy * z_mult_dat[ip] * del_x_slope;
-                             vel_vec = vy;
-                             vel_idx = vy_l;
-
-                             dir = -1;
-                             diff = pp[ip - sy_p] - pp[ip];
-                             u_old = h
-                                     * PMean(pp[ip - sy_p], pp[ip],
-                                             permyp[ip - sy_p], permyp[ip])
-                                     * (diff / dy)
-                                     * RPMean(pp[ip - sy_p], pp[ip],
-                                              rpp[ip - sy_p] * dp[ip - sy_p], rpp[ip] * dp[ip])
-                                     / viscosity;
-
-                             u_old += h *
-                                      PMean(pp[ip], pp[ip - sy_p], permyp[ip],
-                                            permyp[ip - sy_p])
-                                      * (-y_dir_g)
-                                      * RPMean(pp[ip], pp[ip - sy_p], rpp[ip] * dp[ip],
-                                               rpp[ip - sy_p] * dp[ip - sy_p])
-                                      / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffy * z_mult_dat[ip] * del_x_slope;
+        vel_vec = vy;
+        vel_idx = vy_l;
+
+        dir = -1;
+        diff = pp[ip - sy_p] - pp[ip];
+        u_old = h
+                * PMean(pp[ip - sy_p], pp[ip],
+                        permyp[ip - sy_p], permyp[ip])
+                * (diff / dy)
+                * RPMean(pp[ip - sy_p], pp[ip],
+                         rpp[ip - sy_p] * dp[ip - sy_p], rpp[ip] * dp[ip])
+                / viscosity;
+
+        u_old += h *
+                 PMean(pp[ip], pp[ip - sy_p], permyp[ip],
+                       permyp[ip - sy_p])
+                 * (-y_dir_g)
+                 * RPMean(pp[ip], pp[ip - sy_p], rpp[ip] * dp[ip],
+                          rpp[ip - sy_p] * dp[ip - sy_p])
+                 / viscosity;
+        u_new = h;
+      }),
                            FACE(UpFace,
-                           {
-                             h = ffy * z_mult_dat[ip] * del_x_slope;
-                             vel_vec = vy;
-                             vel_idx = vy_l + sy_v;
-
-                             dir = 1;
-                             diff = pp[ip] - pp[ip + sy_p];
-                             u_old = h
-                                     * PMean(pp[ip], pp[ip + sy_p],
-                                             permyp[ip], permyp[ip + sy_p])
-                                     * (diff / dy)
-                                     * RPMean(pp[ip], pp[ip + sy_p],
-                                              rpp[ip] * dp[ip], rpp[ip + sy_p] * dp[ip + sy_p])
-                                     / viscosity;
-
-                             u_old += h
-                                      * PMean(pp[ip], pp[ip + sy_p], permyp[ip],
-                                              permyp[ip + sy_p])
-                                      * (-y_dir_g)
-                                      * RPMean(pp[ip], pp[ip + sy_p], rpp[ip] * dp[ip],
-                                               rpp[ip + sy_p] * dp[ip + sy_p])
-                                      / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffy * z_mult_dat[ip] * del_x_slope;
+        vel_vec = vy;
+        vel_idx = vy_l + sy_v;
+
+        dir = 1;
+        diff = pp[ip] - pp[ip + sy_p];
+        u_old = h
+                * PMean(pp[ip], pp[ip + sy_p],
+                        permyp[ip], permyp[ip + sy_p])
+                * (diff / dy)
+                * RPMean(pp[ip], pp[ip + sy_p],
+                         rpp[ip] * dp[ip], rpp[ip + sy_p] * dp[ip + sy_p])
+                / viscosity;
+
+        u_old += h
+                 * PMean(pp[ip], pp[ip + sy_p], permyp[ip],
+                         permyp[ip + sy_p])
+                 * (-y_dir_g)
+                 * RPMean(pp[ip], pp[ip + sy_p], rpp[ip] * dp[ip],
+                          rpp[ip + sy_p] * dp[ip + sy_p])
+                 / viscosity;
+        u_new = h;
+      }),
                            FACE(BackFace,
-                           {
-                             h = ffz * del_x_slope * del_y_slope;
-                             vel_vec = vz;
-                             vel_idx = vz_l;
-
-                             dir = -1;
-                             sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_p]); //RMM
-                             //  sep = dz*z_mult_dat[ip];  //RMM
-
-                             lower_cond = (pp[ip - sz_p] / sep)
-                                          - (z_mult_dat[ip - sz_p] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip - sz_p] * gravity *
-                                          z_dir_g;
-                             upper_cond = (pp[ip] / sep) + (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip] * gravity *
-                                          z_dir_g;
-
-                             diff = lower_cond - upper_cond;
-                             u_old = h
-                                     * PMeanDZ(permzp[ip - sz_p], permzp[ip],
-                                               z_mult_dat[ip - sz_p], z_mult_dat[ip])
-                                     * diff
-                                     * RPMean(lower_cond, upper_cond,
-                                              rpp[ip - sz_p] * dp[ip - sz_p], rpp[ip] * dp[ip])
-                                     / viscosity;
-                             u_new = h;
-                           }),
+      {
+        h = ffz * del_x_slope * del_y_slope;
+        vel_vec = vz;
+        vel_idx = vz_l;
+
+        dir = -1;
+        sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_p]);                      //RMM
+        //  sep = dz*z_mult_dat[ip];  //RMM
+
+        lower_cond = (pp[ip - sz_p] / sep)
+                     - (z_mult_dat[ip - sz_p] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip - sz_p] * gravity *
+                     z_dir_g;
+        upper_cond = (pp[ip] / sep) + (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip - sz_p])) * dp[ip] * gravity *
+                     z_dir_g;
+
+        diff = lower_cond - upper_cond;
+        u_old = h
+                * PMeanDZ(permzp[ip - sz_p], permzp[ip],
+                          z_mult_dat[ip - sz_p], z_mult_dat[ip])
+                * diff
+                * RPMean(lower_cond, upper_cond,
+                         rpp[ip - sz_p] * dp[ip - sz_p], rpp[ip] * dp[ip])
+                / viscosity;
+        u_new = h;
+      }),
                            FACE(FrontFace,
-                           {
-                             h = ffz * del_x_slope * del_y_slope;
-                             vel_vec = vz;
-                             vel_idx = vz_l + sz_v;
-
-                             dir = 1;
-                             sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_p]); //RMM
-
-                             lower_cond = (pp[ip] / sep) - (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p])) * dp[ip] * gravity *
-                                          z_dir_g;
-                             upper_cond = (pp[ip + sz_p] / sep)
-                                          + (z_mult_dat[ip + sz_p] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p])) * dp[ip + sz_p] * gravity *
-                                          z_dir_g;
-                             diff = lower_cond - upper_cond;
-                             u_old = h
-                                     * PMeanDZ(permzp[ip], permzp[ip + sz_p],
-                                               z_mult_dat[ip], z_mult_dat[ip + sz_p])
-                                     * diff
-                                     * RPMean(lower_cond, upper_cond,
-                                              rpp[ip] * dp[ip], rpp[ip + sz_p] * dp[ip + sz_p])
-                                     / viscosity;
-                             u_new = h;
-
-
-                             q_overlnd = 0.0;
-                             q_overlnd = vol
-                                         * (pfmax(pp[ip], 0.0) - pfmax(opp[ip], 0.0)) / dz +
-                                         dt * vol * ((ke_[io] - kw_[io]) / dx + (kn_[io] - ks_[io]) / dy)
-                                         / dz;
-
-                             fp[ip] += q_overlnd;
-                           }),
+      {
+        h = ffz * del_x_slope * del_y_slope;
+        vel_vec = vz;
+        vel_idx = vz_l + sz_v;
+
+        dir = 1;
+        sep = dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_p]);                      //RMM
+
+        lower_cond = (pp[ip] / sep) - (z_mult_dat[ip] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p])) * dp[ip] * gravity *
+                     z_dir_g;
+        upper_cond = (pp[ip + sz_p] / sep)
+                     + (z_mult_dat[ip + sz_p] / (z_mult_dat[ip] + z_mult_dat[ip + sz_p])) * dp[ip + sz_p] * gravity *
+                     z_dir_g;
+        diff = lower_cond - upper_cond;
+        u_old = h
+                * PMeanDZ(permzp[ip], permzp[ip + sz_p],
+                          z_mult_dat[ip], z_mult_dat[ip + sz_p])
+                * diff
+                * RPMean(lower_cond, upper_cond,
+                         rpp[ip] * dp[ip], rpp[ip + sz_p] * dp[ip + sz_p])
+                / viscosity;
+        u_new = h;
+
+
+        q_overlnd = 0.0;
+        q_overlnd = vol
+                    * (pfmax(pp[ip], 0.0) - pfmax(opp[ip], 0.0)) / dz +
+                    dt * vol * ((ke_[io] - kw_[io]) / dx + (kn_[io] - ks_[io]) / dy)
+                    / dz;
+
+        fp[ip] += q_overlnd;
+      }),
                            CellFinalize(
-                           {
-                             /* Remove the boundary term computed above */
-                             fp[ip] -= dt * dir * u_old;
-                             //add source boundary terms
-                             u_new = u_new * bc_patch_values[ival];       //sk: here we go in and implement surface routing!
-                             fp[ip] += dt * dir * u_new;
+      {
+        /* Remove the boundary term computed above */
+        fp[ip] -= dt * dir * u_old;
+        //add source boundary terms
+        u_new = u_new * bc_patch_values[ival];                            //sk: here we go in and implement surface routing!
+        fp[ip] += dt * dir * u_new;
 
-                             u_new += q_overlnd/dt;
+        u_new += q_overlnd / dt;
 
-                             vel_vec[vel_idx] = u_new / h;
-                           }),
+        vel_vec[vel_idx] = u_new / h;
+      }),
                            AfterAllCells(DoNothing)
-        ); /* End OverlandDiffusiveBC case */
+                           ); /* End OverlandDiffusiveBC case */
     }          /* End ipatch loop */
   }            /* End subgrid loop */
 
@@ -2168,22 +2167,22 @@ void NlFunctionEval(Vector *     pressure, /* Current pressure values */
                            LoopVars(i, j, k, ival, bc_struct, ipatch, is),
                            Locals(int pp_idx, ip; /*double value;*/),
                            CellSetup({
-                               pp_idx = 0;
-                               ip = SubvectorEltIndex(p_sub, i, j, k);
-                               /*value = bc_patch_values[ival];*/
-                             }),
-                           FACE(LeftFace,  { pp_idx = ip - 1; }),
+        pp_idx = 0;
+        ip = SubvectorEltIndex(p_sub, i, j, k);
+        /*value = bc_patch_values[ival];*/
+      }),
+                           FACE(LeftFace, { pp_idx = ip - 1; }),
                            FACE(RightFace, { pp_idx = ip + 1; }),
-                           FACE(DownFace,  { pp_idx = ip - sy_p; }),
-                           FACE(UpFace,    { pp_idx = ip + sy_p; }),
-                           FACE(BackFace,  { pp_idx = ip - sz_p; }),
+                           FACE(DownFace, { pp_idx = ip - sy_p; }),
+                           FACE(UpFace, { pp_idx = ip + sy_p; }),
+                           FACE(BackFace, { pp_idx = ip - sz_p; }),
                            FACE(FrontFace, { pp_idx = ip + sz_p; }),
                            CellFinalize({
-                               pp[pp_idx] = -FLT_MAX;
-                               fp[pp_idx] = 0.0;
-                             }),
+        pp[pp_idx] = -FLT_MAX;
+        fp[pp_idx] = 0.0;
+      }),
                            AfterAllCells(DoNothing)
-        );
+                           );
     }          /* End ipatch loop */
   }            /* End subgrid loop */
 
diff --git a/pfsimulator/parflow_lib/nl_function_eval.h b/pfsimulator/parflow_lib/nl_function_eval.h
index 88210329a..9d9984852 100644
--- a/pfsimulator/parflow_lib/nl_function_eval.h
+++ b/pfsimulator/parflow_lib/nl_function_eval.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 typedef struct {
   PFModule    *nl_function_eval;
diff --git a/pfsimulator/parflow_lib/nodiag_scale.c b/pfsimulator/parflow_lib/nodiag_scale.c
index 81af50f84..725cd241e 100644
--- a/pfsimulator/parflow_lib/nodiag_scale.c
+++ b/pfsimulator/parflow_lib/nodiag_scale.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * FILE: nodiag_scale.c
diff --git a/pfsimulator/parflow_lib/overlandflow_eval.c b/pfsimulator/parflow_lib/overlandflow_eval.c
index 437196a0f..401e51a12 100644
--- a/pfsimulator/parflow_lib/overlandflow_eval.c
+++ b/pfsimulator/parflow_lib/overlandflow_eval.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *  This module computes the contributions for the spatial discretization of the
@@ -83,7 +83,8 @@ void    OverlandFlowEval(
 
   double        *sx_dat, *sy_dat, *mann_dat, *top_dat, *pp;
 
-  int i, j, k, ival=0, sy_v;
+  int i, j, k, ival = 0, sy_v;
+
   PF_UNUSED(ival);
 
   p_sub = VectorSubvector(pressure, sg);
@@ -110,81 +111,81 @@ void    OverlandFlowEval(
                            BeforeAllCells(DoNothing),
                            LoopVars(i, j, k, ival, bc_struct, ipatch, sg),
                            Locals(int io, itop, ip, k1, ii, step;
-                                  double q_v[3], xdir, ydir;),
+                                  double q_v[3], xdir, ydir; ),
                            CellSetup(DoNothing),
                            FACE(LeftFace, DoNothing), FACE(RightFace, DoNothing),
                            FACE(DownFace, DoNothing), FACE(UpFace, DoNothing),
                            FACE(BackFace, DoNothing),
                            FACE(FrontFace,
-                           {
-                             io = SubvectorEltIndex(sx_sub, i, j, 0);
-                             itop = SubvectorEltIndex(top_sub, i, j, 0);
-
-                             /* compute east and west faces */
-                             /* First initialize velocities, q_v, for inactive region */
-                             q_v[0] = 0.0;
-                             q_v[1] = 0.0;
-                             q_v[2] = 0.0;
-                             xdir = 0.0;
-                             ydir = 0.0;
-                             k1 = 0;
-
-                             for (ii = -1; ii < 2; ii++)
-                             {
-                               k1 = (int)top_dat[itop + ii];
-                               if (k1 >= 0)
-                               {
-                                 ip = SubvectorEltIndex(p_sub, (i + ii), j, k1);
-
-                                 if (sx_dat[io + ii] > 0.0)
-                                   xdir = -1.0;
-                                 else if (sx_dat[io + ii] < 0.0)
-                                   xdir = 1.0;
-                                 else
-                                   xdir = 0.0;
-
-                                 q_v[ii + 1] = xdir * (RPowerR(fabs(sx_dat[io + ii]), 0.5) / mann_dat[io + ii])
-                                               * RPowerR(pfmax((pp[ip]), 0.0), (5.0 / 3.0));
-                               }
-                             }
-
-                             /* compute kw and ke - NOTE: io is for current cell */
-                             kw_v[io] = pfmax(q_v[0], 0.0) - pfmax(-q_v[1], 0.0);
-                             ke_v[io] = pfmax(q_v[1], 0.0) - pfmax(-q_v[2], 0.0);
-
-                             /* compute north and south faces */
-                             /* First initialize velocities, q_v, for inactive region */
-                             q_v[0] = 0.0;
-                             q_v[1] = 0.0;
-                             q_v[2] = 0.0;
-
-                             for (ii = -1; ii < 2; ii++)
-                             {
-                               step = ii * sy_v;
-                               k1 = (int)top_dat[itop + step];
-                               if (k1 >= 0)
-                               {
-                                 ip = SubvectorEltIndex(p_sub, i, (j + ii), k1);
-
-                                 if (sy_dat[io + step] > 0.0)
-                                   ydir = -1.0;
-                                 else if (sy_dat[io + step] < 0.0)
-                                   ydir = 1.0;
-                                 else
-                                   ydir = 0.0;
-
-                                 q_v[ii + 1] = ydir * (RPowerR(fabs(sy_dat[io + step]), 0.5) / mann_dat[io + step])
-                                               * RPowerR(pfmax((pp[ip]), 0.0), (5.0 / 3.0));
-                               }
-                             }
-
-                             /* compute ks and kn - NOTE: io is for current cell */
-                             ks_v[io] = pfmax(q_v[0], 0.0) - pfmax(-q_v[1], 0.0);
-                             kn_v[io] = pfmax(q_v[1], 0.0) - pfmax(-q_v[2], 0.0);
-                           }),
+      {
+        io = SubvectorEltIndex(sx_sub, i, j, 0);
+        itop = SubvectorEltIndex(top_sub, i, j, 0);
+
+        /* compute east and west faces */
+        /* First initialize velocities, q_v, for inactive region */
+        q_v[0] = 0.0;
+        q_v[1] = 0.0;
+        q_v[2] = 0.0;
+        xdir = 0.0;
+        ydir = 0.0;
+        k1 = 0;
+
+        for (ii = -1; ii < 2; ii++)
+        {
+          k1 = (int)top_dat[itop + ii];
+          if (k1 >= 0)
+          {
+            ip = SubvectorEltIndex(p_sub, (i + ii), j, k1);
+
+            if (sx_dat[io + ii] > 0.0)
+              xdir = -1.0;
+            else if (sx_dat[io + ii] < 0.0)
+              xdir = 1.0;
+            else
+              xdir = 0.0;
+
+            q_v[ii + 1] = xdir * (RPowerR(fabs(sx_dat[io + ii]), 0.5) / mann_dat[io + ii])
+                          * RPowerR(pfmax((pp[ip]), 0.0), (5.0 / 3.0));
+          }
+        }
+
+        /* compute kw and ke - NOTE: io is for current cell */
+        kw_v[io] = pfmax(q_v[0], 0.0) - pfmax(-q_v[1], 0.0);
+        ke_v[io] = pfmax(q_v[1], 0.0) - pfmax(-q_v[2], 0.0);
+
+        /* compute north and south faces */
+        /* First initialize velocities, q_v, for inactive region */
+        q_v[0] = 0.0;
+        q_v[1] = 0.0;
+        q_v[2] = 0.0;
+
+        for (ii = -1; ii < 2; ii++)
+        {
+          step = ii * sy_v;
+          k1 = (int)top_dat[itop + step];
+          if (k1 >= 0)
+          {
+            ip = SubvectorEltIndex(p_sub, i, (j + ii), k1);
+
+            if (sy_dat[io + step] > 0.0)
+              ydir = -1.0;
+            else if (sy_dat[io + step] < 0.0)
+              ydir = 1.0;
+            else
+              ydir = 0.0;
+
+            q_v[ii + 1] = ydir * (RPowerR(fabs(sy_dat[io + step]), 0.5) / mann_dat[io + step])
+                          * RPowerR(pfmax((pp[ip]), 0.0), (5.0 / 3.0));
+          }
+        }
+
+        /* compute ks and kn - NOTE: io is for current cell */
+        ks_v[io] = pfmax(q_v[0], 0.0) - pfmax(-q_v[1], 0.0);
+        kn_v[io] = pfmax(q_v[1], 0.0) - pfmax(-q_v[2], 0.0);
+      }),
                            CellFinalize(DoNothing),
                            AfterAllCells(DoNothing)
-        );
+                           );
     }
     else   /* return velocity fluxes */
     {
@@ -192,76 +193,76 @@ void    OverlandFlowEval(
                            BeforeAllCells(DoNothing),
                            LoopVars(i, j, k, ival, bc_struct, ipatch, sg),
                            Locals(int io, itop, ip, k1, step, ii;
-                                  double q_v[3], xdir, ydir;),
+                                  double q_v[3], xdir, ydir; ),
                            CellSetup(DoNothing),
                            FACE(LeftFace, DoNothing), FACE(RightFace, DoNothing),
                            FACE(DownFace, DoNothing), FACE(UpFace, DoNothing),
                            FACE(BackFace, DoNothing),
                            FACE(FrontFace,
-                           {
-                             io = SubvectorEltIndex(sx_sub, i, j, 0);
-                             itop = SubvectorEltIndex(top_sub, i, j, 0);
-
-                             /* compute east and west faces */
-                             /* First initialize velocities, q_v, for inactive region */
-                             q_v[0] = 0.0;
-                             q_v[1] = 0.0;
-                             q_v[2] = 0.0;
-
-                             for (ii = -1; ii < 2; ii++)
-                             {
-                               k1 = (int)top_dat[itop + ii];
-                               if (k1 >= 0)
-                               {
-                                 ip = SubvectorEltIndex(p_sub, (i + ii), j, k1);
-
-                                 if (sx_dat[io + ii] > 0.0)
-                                   xdir = -1.0;
-                                 else if (sx_dat[io + ii] < 0.0)
-                                   xdir = 1.0;
-                                 else
-                                   xdir = 0.0;
-
-                                 q_v[ii + 1] = xdir * (RPowerR(fabs(sx_dat[io + ii]), 0.5) / mann_dat[io + ii]) * RPowerR(pfmax((pp[ip]), 0.0), (5.0 / 3.0));
-                               }
-                             }
-                             qx_v[io] = q_v[1];
-                             /* compute kw and ke - NOTE: io is for current cell */
-                             kw_v[io] = pfmax(q_v[0], 0.0) - pfmax(-q_v[1], 0.0);
-                             ke_v[io] = pfmax(q_v[1], 0.0) - pfmax(-q_v[2], 0.0);
-
-                             /* compute north and south faces */
-                             /* First initialize velocities, q_v, for inactive region */
-                             q_v[0] = 0.0;
-                             q_v[1] = 0.0;
-                             q_v[2] = 0.0;
-
-                             for (ii = -1; ii < 2; ii++)
-                             {
-                               step = ii * sy_v;
-                               k1 = (int)top_dat[itop + step];
-                               if (k1 >= 0)
-                               {
-                                 ip = SubvectorEltIndex(p_sub, i, (j + ii), k1);
-
-                                 if (sy_dat[io + step] > 0.0)
-                                   ydir = -1.0;
-                                 else if (sy_dat[io + step] < 0.0)
-                                   ydir = 1.0;
-                                 else
-                                   ydir = 0.0;
-
-                                 q_v[ii + 1] = ydir * (RPowerR(fabs(sy_dat[io + step]), 0.5) / mann_dat[io + step]) * RPowerR(pfmax((pp[ip]), 0.0), (5.0 / 3.0));
-                               }
-                             }
-                             qy_v[io] = q_v[1];
-                             /* compute ks and kn - NOTE: io is for current cell */
-                             ks_v[io] = pfmax(q_v[0], 0.0) - pfmax(-q_v[1], 0.0);
-                             kn_v[io] = pfmax(q_v[1], 0.0) - pfmax(-q_v[2], 0.0);
-                           }),
+      {
+        io = SubvectorEltIndex(sx_sub, i, j, 0);
+        itop = SubvectorEltIndex(top_sub, i, j, 0);
+
+        /* compute east and west faces */
+        /* First initialize velocities, q_v, for inactive region */
+        q_v[0] = 0.0;
+        q_v[1] = 0.0;
+        q_v[2] = 0.0;
+
+        for (ii = -1; ii < 2; ii++)
+        {
+          k1 = (int)top_dat[itop + ii];
+          if (k1 >= 0)
+          {
+            ip = SubvectorEltIndex(p_sub, (i + ii), j, k1);
+
+            if (sx_dat[io + ii] > 0.0)
+              xdir = -1.0;
+            else if (sx_dat[io + ii] < 0.0)
+              xdir = 1.0;
+            else
+              xdir = 0.0;
+
+            q_v[ii + 1] = xdir * (RPowerR(fabs(sx_dat[io + ii]), 0.5) / mann_dat[io + ii]) * RPowerR(pfmax((pp[ip]), 0.0), (5.0 / 3.0));
+          }
+        }
+        qx_v[io] = q_v[1];
+        /* compute kw and ke - NOTE: io is for current cell */
+        kw_v[io] = pfmax(q_v[0], 0.0) - pfmax(-q_v[1], 0.0);
+        ke_v[io] = pfmax(q_v[1], 0.0) - pfmax(-q_v[2], 0.0);
+
+        /* compute north and south faces */
+        /* First initialize velocities, q_v, for inactive region */
+        q_v[0] = 0.0;
+        q_v[1] = 0.0;
+        q_v[2] = 0.0;
+
+        for (ii = -1; ii < 2; ii++)
+        {
+          step = ii * sy_v;
+          k1 = (int)top_dat[itop + step];
+          if (k1 >= 0)
+          {
+            ip = SubvectorEltIndex(p_sub, i, (j + ii), k1);
+
+            if (sy_dat[io + step] > 0.0)
+              ydir = -1.0;
+            else if (sy_dat[io + step] < 0.0)
+              ydir = 1.0;
+            else
+              ydir = 0.0;
+
+            q_v[ii + 1] = ydir * (RPowerR(fabs(sy_dat[io + step]), 0.5) / mann_dat[io + step]) * RPowerR(pfmax((pp[ip]), 0.0), (5.0 / 3.0));
+          }
+        }
+        qy_v[io] = q_v[1];
+        /* compute ks and kn - NOTE: io is for current cell */
+        ks_v[io] = pfmax(q_v[0], 0.0) - pfmax(-q_v[1], 0.0);
+        kn_v[io] = pfmax(q_v[1], 0.0) - pfmax(-q_v[2], 0.0);
+      }),
                            CellFinalize(DoNothing),
                            AfterAllCells(DoNothing)
-        );
+                           );
     }
   }
   else  /* fcn == CALCDER: derivs of KE,KW,KN,KS w.r.t. current cell (i,j,k) */
@@ -272,48 +273,48 @@ void    OverlandFlowEval(
                            BeforeAllCells(DoNothing),
                            LoopVars(i, j, k, ival, bc_struct, ipatch, sg),
                            Locals(int io, ip;
-                                  double xdir, ydir, q_mid;),
+                                  double xdir, ydir, q_mid; ),
                            CellSetup(DoNothing),
                            FACE(LeftFace, DoNothing), FACE(RightFace, DoNothing),
                            FACE(DownFace, DoNothing), FACE(UpFace, DoNothing),
                            FACE(BackFace, DoNothing),
                            FACE(FrontFace,
-                           {
-                             /* compute derivs for east and west faces */
-
-                             /* current cell */
-                             io = SubvectorEltIndex(sx_sub, i, j, 0);
-                             ip = SubvectorEltIndex(p_sub, i, j, k);
-
-                             if (sx_dat[io] > 0.0)
-                               xdir = -1.0;
-                             else if (sx_dat[io] < 0.0)
-                               xdir = 1.0;
-                             else
-                               xdir = 0.0;
-
-                             q_mid = xdir * (5.0 / 3.0) * (RPowerR(fabs(sx_dat[io]), 0.5) / mann_dat[io]) * RPowerR(pfmax((pp[ip]), 0.0), (2.0 / 3.0));
-                             /* compute derivs of kw and ke - NOTE: io is for current cell */
-                             kw_v[io] = -pfmax(-q_mid, 0.0);
-                             ke_v[io] = pfmax(q_mid, 0.0);
-
-
-                             /* compute north and south faces */
-                             if (sy_dat[io] > 0.0)
-                               ydir = -1.0;
-                             else if (sy_dat[io] < 0.0)
-                               ydir = 1.0;
-                             else
-                               ydir = 0.0;
-
-                             q_mid = ydir * (5.0 / 3.0) * (RPowerR(fabs(sy_dat[io]), 0.5) / mann_dat[io]) * RPowerR(pfmax((pp[ip]), 0.0), (2.0 / 3.0));
-                             /* compute derivs of ks and kn - NOTE: io is for current cell */
-                             ks_v[io] = -pfmax(-q_mid, 0.0);
-                             kn_v[io] = pfmax(q_mid, 0.0);
-                           }),
+      {
+        /* compute derivs for east and west faces */
+
+        /* current cell */
+        io = SubvectorEltIndex(sx_sub, i, j, 0);
+        ip = SubvectorEltIndex(p_sub, i, j, k);
+
+        if (sx_dat[io] > 0.0)
+          xdir = -1.0;
+        else if (sx_dat[io] < 0.0)
+          xdir = 1.0;
+        else
+          xdir = 0.0;
+
+        q_mid = xdir * (5.0 / 3.0) * (RPowerR(fabs(sx_dat[io]), 0.5) / mann_dat[io]) * RPowerR(pfmax((pp[ip]), 0.0), (2.0 / 3.0));
+        /* compute derivs of kw and ke - NOTE: io is for current cell */
+        kw_v[io] = -pfmax(-q_mid, 0.0);
+        ke_v[io] = pfmax(q_mid, 0.0);
+
+
+        /* compute north and south faces */
+        if (sy_dat[io] > 0.0)
+          ydir = -1.0;
+        else if (sy_dat[io] < 0.0)
+          ydir = 1.0;
+        else
+          ydir = 0.0;
+
+        q_mid = ydir * (5.0 / 3.0) * (RPowerR(fabs(sy_dat[io]), 0.5) / mann_dat[io]) * RPowerR(pfmax((pp[ip]), 0.0), (2.0 / 3.0));
+        /* compute derivs of ks and kn - NOTE: io is for current cell */
+        ks_v[io] = -pfmax(-q_mid, 0.0);
+        kn_v[io] = pfmax(q_mid, 0.0);
+      }),
                            CellFinalize(DoNothing),
                            AfterAllCells(DoNothing)
-        );
+                           );
     }
     else   /* return derivs of velocity fluxes */
     {
@@ -321,50 +322,50 @@ void    OverlandFlowEval(
                            BeforeAllCells(DoNothing),
                            LoopVars(i, j, k, ival, bc_struct, ipatch, sg),
                            Locals(int io, ip;
-                                  double xdir, ydir, q_mid;),
+                                  double xdir, ydir, q_mid; ),
                            CellSetup(DoNothing),
                            FACE(LeftFace, DoNothing), FACE(RightFace, DoNothing),
                            FACE(DownFace, DoNothing), FACE(UpFace, DoNothing),
                            FACE(BackFace, DoNothing),
                            FACE(FrontFace,
-                           {
-                             /* compute derivs for east and west faces */
-
-                             /* current cell */
-                             io = SubvectorEltIndex(sx_sub, i, j, 0);
-                             ip = SubvectorEltIndex(p_sub, i, j, k);
-
-                             if (sx_dat[io] > 0.0)
-                               xdir = -1.0;
-                             else if (sx_dat[io] < 0.0)
-                               xdir = 1.0;
-                             else
-                               xdir = 0.0;
-
-                             q_mid = xdir * (5.0 / 3.0) * (RPowerR(fabs(sx_dat[io]), 0.5) / mann_dat[io]) * RPowerR(pfmax((pp[ip]), 0.0), (2.0 / 3.0));
-                             qx_v[io] = q_mid;
-                             /* compute derivs of kw and ke - NOTE: io is for current cell */
-                             kw_v[io] = -pfmax(-q_mid, 0.0);
-                             ke_v[io] = pfmax(q_mid, 0.0);
-
-
-                             /* compute north and south faces */
-                             if (sy_dat[io] > 0.0)
-                               ydir = -1.0;
-                             else if (sy_dat[io] < 0.0)
-                               ydir = 1.0;
-                             else
-                               ydir = 0.0;
-
-                             q_mid = ydir * (5.0 / 3.0) * (RPowerR(fabs(sy_dat[io]), 0.5) / mann_dat[io]) * RPowerR(pfmax((pp[ip]), 0.0), (2.0 / 3.0));
-                             qy_v[io] = q_mid;
-                             /* compute derivs of ks and kn - NOTE: io is for current cell */
-                             ks_v[io] = -pfmax(-q_mid, 0.0);
-                             kn_v[io] = pfmax(q_mid, 0.0);
-                           }),
+      {
+        /* compute derivs for east and west faces */
+
+        /* current cell */
+        io = SubvectorEltIndex(sx_sub, i, j, 0);
+        ip = SubvectorEltIndex(p_sub, i, j, k);
+
+        if (sx_dat[io] > 0.0)
+          xdir = -1.0;
+        else if (sx_dat[io] < 0.0)
+          xdir = 1.0;
+        else
+          xdir = 0.0;
+
+        q_mid = xdir * (5.0 / 3.0) * (RPowerR(fabs(sx_dat[io]), 0.5) / mann_dat[io]) * RPowerR(pfmax((pp[ip]), 0.0), (2.0 / 3.0));
+        qx_v[io] = q_mid;
+        /* compute derivs of kw and ke - NOTE: io is for current cell */
+        kw_v[io] = -pfmax(-q_mid, 0.0);
+        ke_v[io] = pfmax(q_mid, 0.0);
+
+
+        /* compute north and south faces */
+        if (sy_dat[io] > 0.0)
+          ydir = -1.0;
+        else if (sy_dat[io] < 0.0)
+          ydir = 1.0;
+        else
+          ydir = 0.0;
+
+        q_mid = ydir * (5.0 / 3.0) * (RPowerR(fabs(sy_dat[io]), 0.5) / mann_dat[io]) * RPowerR(pfmax((pp[ip]), 0.0), (2.0 / 3.0));
+        qy_v[io] = q_mid;
+        /* compute derivs of ks and kn - NOTE: io is for current cell */
+        ks_v[io] = -pfmax(-q_mid, 0.0);
+        kn_v[io] = pfmax(q_mid, 0.0);
+      }),
                            CellFinalize(DoNothing),
                            AfterAllCells(DoNothing)
-        );
+                           );
     }
   }
 }
@@ -437,4 +438,4 @@ void  OverlandFlowEvalFreePublicXtra()
 int  OverlandFlowEvalSizeOfTempData()
 {
   return 0;
-}
+}
\ No newline at end of file
diff --git a/pfsimulator/parflow_lib/overlandflow_eval_Kin.c b/pfsimulator/parflow_lib/overlandflow_eval_Kin.c
index 952db28f7..73e503823 100644
--- a/pfsimulator/parflow_lib/overlandflow_eval_Kin.c
+++ b/pfsimulator/parflow_lib/overlandflow_eval_Kin.c
@@ -1,18 +1,30 @@
-/*BHEADER*********************************************************************
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *  This module computes the contributions for the spatial discretization of the
@@ -78,7 +90,8 @@ void    OverlandFlowEvalKin(
 
   double ov_epsilon;
 
-  int i, j, k, ival=0, sy_v;
+  int i, j, k, ival = 0, sy_v;
+
   PF_UNUSED(ival);
 
   p_sub = VectorSubvector(pressure, sg);
@@ -109,112 +122,112 @@ void    OverlandFlowEvalKin(
                                   Locals(int io, itop, ip, ipp1, ippsy;
                                          int k1, k0x, k0y, k1x, k1y;
                                          double Sf_x, Sf_y, Sf_mag;
-                                         double Press_x, Press_y;),
+                                         double Press_x, Press_y; ),
                                   CellSetup(DoNothing),
                                   FACE(LeftFace, DoNothing), FACE(RightFace, DoNothing),
                                   FACE(DownFace, DoNothing), FACE(UpFace, DoNothing),
                                   FACE(BackFace, DoNothing),
                                   FACE(FrontFace,
-                                  {
-                                    io = SubvectorEltIndex(sx_sub, i, j, 0);
-                                    itop = SubvectorEltIndex(top_sub, i, j, 0);
-
-                                    k1 = (int)top_dat[itop];
-                                    k0x = (int)top_dat[itop - 1];
-                                    k0y = (int)top_dat[itop - sy_v];
-                                    k1x = pfmax((int)top_dat[itop + 1],0);
-                                    k1y = pfmax((int)top_dat[itop + sy_v],0);
-
-                                    if (k1 >= 0)
-                                    {
-                                      ip = SubvectorEltIndex(p_sub, i, j, k1);
-                                      Sf_x = sx_dat[io];
-                                      Sf_y = sy_dat[io];
-                                      ipp1 = (int)SubvectorEltIndex(p_sub, i+1, j, k1x);
-                                      ippsy = (int)SubvectorEltIndex(p_sub, i, j+1, k1y);
-
-                                      Sf_mag = RPowerR(Sf_x * Sf_x + Sf_y * Sf_y, 0.5);
-                                      if (Sf_mag < ov_epsilon)
-                                        Sf_mag = ov_epsilon;
-
-                                      Press_x = RPMean(-Sf_x, 0.0,
-                                                       pfmax((pp[ip]), 0.0),
-                                                       pfmax((pp[ipp1]), 0.0));
-                                      Press_y = RPMean(-Sf_y, 0.0,
-                                                       pfmax((pp[ip]), 0.0),
-                                                       pfmax((pp[ippsy]), 0.0));
-
-                                      qx_v[io] = -(Sf_x / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io]))
-                                                 * RPowerR(Press_x, (5.0 / 3.0));
-                                      qy_v[io] = -(Sf_y / (RPowerR(fabs(Sf_mag), 0.5)
-                                                           * mann_dat[io])) * RPowerR(Press_y, (5.0 / 3.0));
-                                    }
-
-                                    //fix for lower x boundary
-                                    if (k0x < 0.0)
-                                    {
-                                      if (k1 >= 0.0)
-                                      {
-                                        Sf_x = sx_dat[io];
-                                        Sf_y = sy_dat[io];
-
-                                        double Sf_mag = RPowerR(Sf_x * Sf_x + Sf_y * Sf_y, 0.5);
-                                        if (Sf_mag < ov_epsilon)
-                                          Sf_mag = ov_epsilon;
-
-                                        if (Sf_x > 0.0)
-                                        {
-                                          ip = SubvectorEltIndex(p_sub, i, j, k1);
-                                          Press_x = pfmax((pp[ip]), 0.0);
-                                          qx_v[io - 1] = -(Sf_x / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_x, (5.0 / 3.0));
-                                        }
-                                      }
-                                    }
-
-                                    //fix for lower y boundary
-                                    if (k0y < 0.0)
-                                    {
-                                      if (k1 >= 0.0)
-                                      {
-                                        Sf_x = sx_dat[io];
-                                        Sf_y = sy_dat[io];
-
-                                        double Sf_mag = RPowerR(Sf_x * Sf_x + Sf_y * Sf_y, 0.5);
-                                        if (Sf_mag < ov_epsilon)
-                                          Sf_mag = ov_epsilon;
-
-                                        if (Sf_y > 0.0)
-                                        {
-                                          ip = SubvectorEltIndex(p_sub, i, j, k1);
-                                          Press_y = pfmax((pp[ip]), 0.0);
-                                          qy_v[io - sy_v] = -(Sf_y / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_y, (5.0 / 3.0));
-                                        }
-                                      }
-                                    }
-                                  }),
+    {
+      io = SubvectorEltIndex(sx_sub, i, j, 0);
+      itop = SubvectorEltIndex(top_sub, i, j, 0);
+
+      k1 = (int)top_dat[itop];
+      k0x = (int)top_dat[itop - 1];
+      k0y = (int)top_dat[itop - sy_v];
+      k1x = pfmax((int)top_dat[itop + 1], 0);
+      k1y = pfmax((int)top_dat[itop + sy_v], 0);
+
+      if (k1 >= 0)
+      {
+        ip = SubvectorEltIndex(p_sub, i, j, k1);
+        Sf_x = sx_dat[io];
+        Sf_y = sy_dat[io];
+        ipp1 = (int)SubvectorEltIndex(p_sub, i + 1, j, k1x);
+        ippsy = (int)SubvectorEltIndex(p_sub, i, j + 1, k1y);
+
+        Sf_mag = RPowerR(Sf_x * Sf_x + Sf_y * Sf_y, 0.5);
+        if (Sf_mag < ov_epsilon)
+          Sf_mag = ov_epsilon;
+
+        Press_x = RPMean(-Sf_x, 0.0,
+                         pfmax((pp[ip]), 0.0),
+                         pfmax((pp[ipp1]), 0.0));
+        Press_y = RPMean(-Sf_y, 0.0,
+                         pfmax((pp[ip]), 0.0),
+                         pfmax((pp[ippsy]), 0.0));
+
+        qx_v[io] = -(Sf_x / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io]))
+                   * RPowerR(Press_x, (5.0 / 3.0));
+        qy_v[io] = -(Sf_y / (RPowerR(fabs(Sf_mag), 0.5)
+                             * mann_dat[io])) * RPowerR(Press_y, (5.0 / 3.0));
+      }
+
+      //fix for lower x boundary
+      if (k0x < 0.0)
+      {
+        if (k1 >= 0.0)
+        {
+          Sf_x = sx_dat[io];
+          Sf_y = sy_dat[io];
+
+          double Sf_mag = RPowerR(Sf_x * Sf_x + Sf_y * Sf_y, 0.5);
+          if (Sf_mag < ov_epsilon)
+            Sf_mag = ov_epsilon;
+
+          if (Sf_x > 0.0)
+          {
+            ip = SubvectorEltIndex(p_sub, i, j, k1);
+            Press_x = pfmax((pp[ip]), 0.0);
+            qx_v[io - 1] = -(Sf_x / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_x, (5.0 / 3.0));
+          }
+        }
+      }
+
+      //fix for lower y boundary
+      if (k0y < 0.0)
+      {
+        if (k1 >= 0.0)
+        {
+          Sf_x = sx_dat[io];
+          Sf_y = sy_dat[io];
+
+          double Sf_mag = RPowerR(Sf_x * Sf_x + Sf_y * Sf_y, 0.5);
+          if (Sf_mag < ov_epsilon)
+            Sf_mag = ov_epsilon;
+
+          if (Sf_y > 0.0)
+          {
+            ip = SubvectorEltIndex(p_sub, i, j, k1);
+            Press_y = pfmax((pp[ip]), 0.0);
+            qy_v[io - sy_v] = -(Sf_y / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_y, (5.0 / 3.0));
+          }
+        }
+      }
+    }),
                                   CellFinalize(DoNothing),
                                   AfterAllCells(DoNothing)
-      );
+                                  );
 
     ForPatchCellsPerFace(BC_ALL,
                          BeforeAllCells(DoNothing),
                          LoopVars(i, j, k, ival, bc_struct, ipatch, sg),
-                         Locals(int io;),
+                         Locals(int io; ),
                          CellSetup(DoNothing),
                          FACE(LeftFace, DoNothing), FACE(RightFace, DoNothing),
                          FACE(DownFace, DoNothing), FACE(UpFace, DoNothing),
                          FACE(BackFace, DoNothing),
                          FACE(FrontFace,
-                         {
-                           io = SubvectorEltIndex(sx_sub, i, j, 0);
-                           ke_v[io] = qx_v[io];
-                           kw_v[io] = qx_v[io - 1];
-                           kn_v[io] = qy_v[io];
-                           ks_v[io] = qy_v[io - sy_v];
-                         }),
+    {
+      io = SubvectorEltIndex(sx_sub, i, j, 0);
+      ke_v[io] = qx_v[io];
+      kw_v[io] = qx_v[io - 1];
+      kn_v[io] = qy_v[io];
+      ks_v[io] = qy_v[io - sy_v];
+    }),
                          CellFinalize(DoNothing),
                          AfterAllCells(DoNothing)
-      );
+                         );
   }
   else          //fcn = CALCDER calculates the derivs
   {
@@ -224,102 +237,102 @@ void    OverlandFlowEvalKin(
                                   Locals(int io, itop, ip, ipp1, ippsy;
                                          int k1, k0x, k0y, k1x, k1y;
                                          double Sf_x, Sf_y, Sf_mag;
-                                         double Press_x, Press_y, qx_temp, qy_temp;),
+                                         double Press_x, Press_y, qx_temp, qy_temp; ),
                                   CellSetup(DoNothing),
                                   FACE(LeftFace, DoNothing), FACE(RightFace, DoNothing),
                                   FACE(DownFace, DoNothing), FACE(UpFace, DoNothing),
                                   FACE(BackFace, DoNothing),
                                   FACE(FrontFace,
-                                  {
-                                    io = SubvectorEltIndex(sx_sub, i, j, 0);
-                                    itop = SubvectorEltIndex(top_sub, i, j, 0);
-
-                                    k1 = (int)top_dat[itop];
-                                    k0x = (int)top_dat[itop - 1];
-                                    k0y = (int)top_dat[itop - sy_v];
-                                    k1x = (int)top_dat[itop + 1];
-                                    k1y = (int)top_dat[itop + sy_v];
-
-                                    if (k1 >= 0)
-                                    {
-                                      ip = SubvectorEltIndex(p_sub, i, j, k1);
-                                      ipp1 = (int)SubvectorEltIndex(p_sub, i+1, j, k1x);
-                                      ippsy = (int)SubvectorEltIndex(p_sub, i, j+1, k1y);
-
-                                      Sf_x = sx_dat[io];
-                                      Sf_y = sy_dat[io];
-
-                                      Sf_mag = RPowerR(Sf_x * Sf_x + Sf_y * Sf_y, 0.5);
-                                      if (Sf_mag < ov_epsilon)
-                                        Sf_mag = ov_epsilon;
-
-                                      Press_x = RPMean(-Sf_x, 0.0,
-                                                       pfmax((pp[ip]), 0.0),
-                                                       pfmax((pp[ipp1]), 0.0));
-                                      Press_y = RPMean(-Sf_y, 0.0,
-                                                       pfmax((pp[ip]), 0.0),
-                                                       pfmax((pp[ippsy]), 0.0));
-
-                                      qx_temp = -(5.0 / 3.0) * (Sf_x / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_x, (2.0 / 3.0));
-                                      qy_temp = -(5.0 / 3.0) * (Sf_y / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_y, (2.0 / 3.0));
-
-                                      ke_v[io] = pfmax(qx_temp, 0);
-                                      kw_v[io + 1] = -pfmax(-qx_temp, 0);
-                                      kn_v[io] = pfmax(qy_temp, 0);
-                                      ks_v[io + sy_v] = -pfmax(-qy_temp, 0);
-                                    }
-
-                                    //fix for lower x boundary
-                                    if (k0x < 0.0)
-                                    {
-                                      if (k1 >= 0.0)
-                                      {
-                                        Sf_x = sx_dat[io];
-                                        Sf_y = sy_dat[io];
-
-                                        double Sf_mag = RPowerR(Sf_x * Sf_x + Sf_y * Sf_y, 0.5);
-                                        if (Sf_mag < ov_epsilon)
-                                          Sf_mag = ov_epsilon;
-
-                                        if (Sf_x > 0.0)
-                                        {
-                                          ip = SubvectorEltIndex(p_sub, i, j, k1);
-                                          Press_x = pfmax((pp[ip]), 0.0);
-                                          qx_temp = -(5.0 / 3.0) * (Sf_x / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_x, (2.0 / 3.0));
-
-                                          kw_v[io] = qx_temp;
-                                          ke_v[io - 1] = qx_temp;
-                                        }
-                                      }
-                                    }
-
-                                    //fix for lower y boundary
-                                    if (k0y < 0.0)
-                                    {
-                                      if (k1 >= 0.0)
-                                      {
-                                        Sf_x = sx_dat[io];
-                                        Sf_y = sy_dat[io];
-
-                                        double Sf_mag = RPowerR(Sf_x * Sf_x + Sf_y * Sf_y, 0.5);  //+ov_epsilon;
-                                        if (Sf_mag < ov_epsilon)
-                                          Sf_mag = ov_epsilon;
-
-                                        if (Sf_y > 0.0)
-                                        {
-                                          ip = SubvectorEltIndex(p_sub, i, j, k1);
-                                          Press_y = pfmax((pp[ip]), 0.0);
-                                          qy_temp = -(5.0 / 3.0) * (Sf_y / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_y, (2.0 / 3.0));
-
-                                          ks_v[io] = qy_temp;
-                                          kn_v[io - sy_v] = qy_temp;
-                                        }
-                                      }
-                                    }
-                                  }),
+    {
+      io = SubvectorEltIndex(sx_sub, i, j, 0);
+      itop = SubvectorEltIndex(top_sub, i, j, 0);
+
+      k1 = (int)top_dat[itop];
+      k0x = (int)top_dat[itop - 1];
+      k0y = (int)top_dat[itop - sy_v];
+      k1x = (int)top_dat[itop + 1];
+      k1y = (int)top_dat[itop + sy_v];
+
+      if (k1 >= 0)
+      {
+        ip = SubvectorEltIndex(p_sub, i, j, k1);
+        ipp1 = (int)SubvectorEltIndex(p_sub, i + 1, j, k1x);
+        ippsy = (int)SubvectorEltIndex(p_sub, i, j + 1, k1y);
+
+        Sf_x = sx_dat[io];
+        Sf_y = sy_dat[io];
+
+        Sf_mag = RPowerR(Sf_x * Sf_x + Sf_y * Sf_y, 0.5);
+        if (Sf_mag < ov_epsilon)
+          Sf_mag = ov_epsilon;
+
+        Press_x = RPMean(-Sf_x, 0.0,
+                         pfmax((pp[ip]), 0.0),
+                         pfmax((pp[ipp1]), 0.0));
+        Press_y = RPMean(-Sf_y, 0.0,
+                         pfmax((pp[ip]), 0.0),
+                         pfmax((pp[ippsy]), 0.0));
+
+        qx_temp = -(5.0 / 3.0) * (Sf_x / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_x, (2.0 / 3.0));
+        qy_temp = -(5.0 / 3.0) * (Sf_y / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_y, (2.0 / 3.0));
+
+        ke_v[io] = pfmax(qx_temp, 0);
+        kw_v[io + 1] = -pfmax(-qx_temp, 0);
+        kn_v[io] = pfmax(qy_temp, 0);
+        ks_v[io + sy_v] = -pfmax(-qy_temp, 0);
+      }
+
+      //fix for lower x boundary
+      if (k0x < 0.0)
+      {
+        if (k1 >= 0.0)
+        {
+          Sf_x = sx_dat[io];
+          Sf_y = sy_dat[io];
+
+          double Sf_mag = RPowerR(Sf_x * Sf_x + Sf_y * Sf_y, 0.5);
+          if (Sf_mag < ov_epsilon)
+            Sf_mag = ov_epsilon;
+
+          if (Sf_x > 0.0)
+          {
+            ip = SubvectorEltIndex(p_sub, i, j, k1);
+            Press_x = pfmax((pp[ip]), 0.0);
+            qx_temp = -(5.0 / 3.0) * (Sf_x / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_x, (2.0 / 3.0));
+
+            kw_v[io] = qx_temp;
+            ke_v[io - 1] = qx_temp;
+          }
+        }
+      }
+
+      //fix for lower y boundary
+      if (k0y < 0.0)
+      {
+        if (k1 >= 0.0)
+        {
+          Sf_x = sx_dat[io];
+          Sf_y = sy_dat[io];
+
+          double Sf_mag = RPowerR(Sf_x * Sf_x + Sf_y * Sf_y, 0.5);                                //+ov_epsilon;
+          if (Sf_mag < ov_epsilon)
+            Sf_mag = ov_epsilon;
+
+          if (Sf_y > 0.0)
+          {
+            ip = SubvectorEltIndex(p_sub, i, j, k1);
+            Press_y = pfmax((pp[ip]), 0.0);
+            qy_temp = -(5.0 / 3.0) * (Sf_y / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_y, (2.0 / 3.0));
+
+            ks_v[io] = qy_temp;
+            kn_v[io - sy_v] = qy_temp;
+          }
+        }
+      }
+    }),
                                   CellFinalize(DoNothing),
                                   AfterAllCells(DoNothing)
-      );
+                                  );
   }   // else calcder
 }     // function
 
diff --git a/pfsimulator/parflow_lib/overlandflow_eval_diffusive.c b/pfsimulator/parflow_lib/overlandflow_eval_diffusive.c
index 16609d7d3..0b752a3a5 100644
--- a/pfsimulator/parflow_lib/overlandflow_eval_diffusive.c
+++ b/pfsimulator/parflow_lib/overlandflow_eval_diffusive.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *  This module computes the contributions for the spatial discretization of the
@@ -96,6 +96,7 @@ void    OverlandFlowEvalDiff(
   double dx, dy, ov_epsilon;
 
   int i, j, k, ival, sy_v;
+
   PF_UNUSED(ival);
 
   p_sub = VectorSubvector(pressure, sg);
@@ -126,307 +127,307 @@ void    OverlandFlowEvalDiff(
     ForPatchCellsPerFaceWithGhost(BC_ALL,
                                   BeforeAllCells(DoNothing),
                                   LoopVars(i, j, k, ival, bc_struct, ipatch, sg),
-                                  Locals(int io, itop, ip=0, ipp1, ippsy;
+                                  Locals(int io, itop, ip = 0, ipp1, ippsy;
                                          int k1, k0x, k0y, k1x, k1y;
-                                         double Press_x=NAN, Press_y;
-                                         double Sf_x=NAN, Sf_y, Sf_xo=NAN, Sf_yo=NAN, Sf_mag;
-                                         double Pupx=NAN, Pupy, Pupox, Pupoy, Pdown, Pdowno;),
+                                         double Press_x = NAN, Press_y;
+                                         double Sf_x = NAN, Sf_y, Sf_xo = NAN, Sf_yo = NAN, Sf_mag;
+                                         double Pupx = NAN, Pupy, Pupox, Pupoy, Pdown, Pdowno; ),
                                   CellSetup(DoNothing),
                                   FACE(LeftFace, DoNothing), FACE(RightFace, DoNothing),
                                   FACE(DownFace, DoNothing), FACE(UpFace, DoNothing),
                                   FACE(BackFace, DoNothing),
                                   FACE(FrontFace,
-                                  {
-                                    io = SubvectorEltIndex(sx_sub, i, j, 0);
-                                    itop = SubvectorEltIndex(top_sub, i, j, 0);
-
-                                    k1 = (int)top_dat[itop];
-                                    k0x = (int)top_dat[itop - 1];
-                                    k0y = (int)top_dat[itop - sy_v];
-                                    k1x = (int)top_dat[itop + 1];
-                                    k1y = (int)top_dat[itop + sy_v];
-
-                                    if (k1 >= 0)
-                                    {
-                                      ip = SubvectorEltIndex(p_sub, i, j, k1);
-                                      ipp1 = (int)SubvectorEltIndex(p_sub, i+1, j, k1x);
-                                      ippsy = (int)SubvectorEltIndex(p_sub, i, j+1, k1y);
-                                      Pupx = pfmax(pp[ipp1], 0.0);
-                                      Pupy = pfmax(pp[ippsy], 0.0);
-                                      Pupox = pfmax(opp[ipp1], 0.0);
-                                      Pupoy = pfmax(opp[ippsy], 0.0);
-                                      Pdown = pfmax(pp[ip], 0.0);
-                                      Pdowno = pfmax(opp[ip], 0.0);
-
-                                      Sf_x = sx_dat[io] + (Pupx - Pdown) / dx;
-                                      Sf_y = sy_dat[io] + (Pupy - Pdown) / dy;
-
-                                      Sf_xo = sx_dat[io] + (Pupox - Pdowno) / dx;
-                                      Sf_yo = sy_dat[io] + (Pupoy - Pdowno) / dy;
-
-                                      Sf_mag = RPowerR(Sf_xo * Sf_xo + Sf_yo * Sf_yo, 0.5);
-                                      if (Sf_mag < ov_epsilon)
-                                        Sf_mag = ov_epsilon;
-
-                                      Press_x = RPMean(-Sf_x, 0.0, pfmax((pp[ip]), 0.0), pfmax((pp[ipp1]), 0.0));
-                                      Press_y = RPMean(-Sf_y, 0.0, pfmax((pp[ip]), 0.0), pfmax((pp[ippsy]), 0.0));
-
-                                      qx_v[io] = -(Sf_x / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_x, (5.0 / 3.0));
-                                      qy_v[io] = -(Sf_y / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_y, (5.0 / 3.0));
-                                    }
-
-                                    //fix for lower x boundary
-                                    if (k0x < 0.0)
-                                    {
-                                      Press_x = pfmax((pp[ip]), 0.0);
-                                      Sf_x = sx_dat[io] + (Press_x - 0.0) / dx;
-
-                                      Pupox = pfmax(opp[ip], 0.0);
-                                      Sf_xo = sx_dat[io] + (Pupox - 0.0) / dx;
-
-                                      double Sf_mag = RPowerR(Sf_xo * Sf_xo + Sf_yo * Sf_yo, 0.5); //+ov_epsilon;
-                                      if (Sf_mag < ov_epsilon)
-                                        Sf_mag = ov_epsilon;
-                                      if (Sf_x > 0.0)
-                                      {
-                                        qx_v[io - 1] = -(Sf_x / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_x, (5.0 / 3.0));
-                                      }
-                                    }
-
-                                    //fix for lower y boundary
-                                    if (k0y < 0.0)
-                                    {
-                                      Press_y = pfmax((pp[ip]), 0.0);
-                                      Sf_y = sy_dat[io] + (Press_y - 0.0) / dx;
-
-                                      Pupoy = pfmax(opp[ip], 0.0);
-                                      Sf_yo = sy_dat[io] + (Pupoy - 0.0) / dx;
-
-                                      double Sf_mag = RPowerR(Sf_xo * Sf_xo + Sf_yo * Sf_yo, 0.5); //Note that the sf_xo was already corrected above
-                                      if (Sf_mag < ov_epsilon)
-                                        Sf_mag = ov_epsilon;
-
-                                      if (Sf_y > 0.0)
-                                      {
-                                        qy_v[io - sy_v] = -(Sf_y / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_y, (5.0 / 3.0));
-                                      }
-
-                                      // Recalculating the x flow in the case with both the lower and left boundaries
-                                      // This is exactly the same as the q_x in the left boundary conditional above but
-                                      // recalculating qx_v here again becuase the sf_mag will be adjusted with the new sf_yo above
-                                      if (k0x < 0.0)
-                                      {
-                                        if (Sf_x > 0.0)
-                                        {
-                                          qx_v[io - 1] = -(Sf_x / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_x, (5.0 / 3.0));
-                                        }
-                                      }
-                                    }
-                                  }),
+    {
+      io = SubvectorEltIndex(sx_sub, i, j, 0);
+      itop = SubvectorEltIndex(top_sub, i, j, 0);
+
+      k1 = (int)top_dat[itop];
+      k0x = (int)top_dat[itop - 1];
+      k0y = (int)top_dat[itop - sy_v];
+      k1x = (int)top_dat[itop + 1];
+      k1y = (int)top_dat[itop + sy_v];
+
+      if (k1 >= 0)
+      {
+        ip = SubvectorEltIndex(p_sub, i, j, k1);
+        ipp1 = (int)SubvectorEltIndex(p_sub, i + 1, j, k1x);
+        ippsy = (int)SubvectorEltIndex(p_sub, i, j + 1, k1y);
+        Pupx = pfmax(pp[ipp1], 0.0);
+        Pupy = pfmax(pp[ippsy], 0.0);
+        Pupox = pfmax(opp[ipp1], 0.0);
+        Pupoy = pfmax(opp[ippsy], 0.0);
+        Pdown = pfmax(pp[ip], 0.0);
+        Pdowno = pfmax(opp[ip], 0.0);
+
+        Sf_x = sx_dat[io] + (Pupx - Pdown) / dx;
+        Sf_y = sy_dat[io] + (Pupy - Pdown) / dy;
+
+        Sf_xo = sx_dat[io] + (Pupox - Pdowno) / dx;
+        Sf_yo = sy_dat[io] + (Pupoy - Pdowno) / dy;
+
+        Sf_mag = RPowerR(Sf_xo * Sf_xo + Sf_yo * Sf_yo, 0.5);
+        if (Sf_mag < ov_epsilon)
+          Sf_mag = ov_epsilon;
+
+        Press_x = RPMean(-Sf_x, 0.0, pfmax((pp[ip]), 0.0), pfmax((pp[ipp1]), 0.0));
+        Press_y = RPMean(-Sf_y, 0.0, pfmax((pp[ip]), 0.0), pfmax((pp[ippsy]), 0.0));
+
+        qx_v[io] = -(Sf_x / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_x, (5.0 / 3.0));
+        qy_v[io] = -(Sf_y / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_y, (5.0 / 3.0));
+      }
+
+      //fix for lower x boundary
+      if (k0x < 0.0)
+      {
+        Press_x = pfmax((pp[ip]), 0.0);
+        Sf_x = sx_dat[io] + (Press_x - 0.0) / dx;
+
+        Pupox = pfmax(opp[ip], 0.0);
+        Sf_xo = sx_dat[io] + (Pupox - 0.0) / dx;
+
+        double Sf_mag = RPowerR(Sf_xo * Sf_xo + Sf_yo * Sf_yo, 0.5);                               //+ov_epsilon;
+        if (Sf_mag < ov_epsilon)
+          Sf_mag = ov_epsilon;
+        if (Sf_x > 0.0)
+        {
+          qx_v[io - 1] = -(Sf_x / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_x, (5.0 / 3.0));
+        }
+      }
+
+      //fix for lower y boundary
+      if (k0y < 0.0)
+      {
+        Press_y = pfmax((pp[ip]), 0.0);
+        Sf_y = sy_dat[io] + (Press_y - 0.0) / dx;
+
+        Pupoy = pfmax(opp[ip], 0.0);
+        Sf_yo = sy_dat[io] + (Pupoy - 0.0) / dx;
+
+        double Sf_mag = RPowerR(Sf_xo * Sf_xo + Sf_yo * Sf_yo, 0.5);                               //Note that the sf_xo was already corrected above
+        if (Sf_mag < ov_epsilon)
+          Sf_mag = ov_epsilon;
+
+        if (Sf_y > 0.0)
+        {
+          qy_v[io - sy_v] = -(Sf_y / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_y, (5.0 / 3.0));
+        }
+
+        // Recalculating the x flow in the case with both the lower and left boundaries
+        // This is exactly the same as the q_x in the left boundary conditional above but
+        // recalculating qx_v here again becuase the sf_mag will be adjusted with the new sf_yo above
+        if (k0x < 0.0)
+        {
+          if (Sf_x > 0.0)
+          {
+            qx_v[io - 1] = -(Sf_x / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io])) * RPowerR(Press_x, (5.0 / 3.0));
+          }
+        }
+      }
+    }),
                                   CellFinalize(DoNothing),
                                   AfterAllCells(DoNothing)
-      );
+                                  );
 
     ForPatchCellsPerFace(BC_ALL,
                          BeforeAllCells(DoNothing),
                          LoopVars(i, j, k, ival, bc_struct, ipatch, sg),
-                         Locals(int io;),
+                         Locals(int io; ),
                          CellSetup(DoNothing),
                          FACE(LeftFace, DoNothing), FACE(RightFace, DoNothing),
                          FACE(DownFace, DoNothing), FACE(UpFace, DoNothing),
                          FACE(BackFace, DoNothing),
                          FACE(FrontFace,
-                         {
-                           io = SubvectorEltIndex(sx_sub, i, j, 0);
-                           ke_v[io] = qx_v[io];
-                           kw_v[io] = qx_v[io - 1];
-                           kn_v[io] = qy_v[io];
-                           ks_v[io] = qy_v[io - sy_v];
-                           //printf("i=%d j=%d k=%d ke_v=%d kw_v=%d kn_v=%d ks_v=%f\n",i,j,k,ke_v[io],kw_v[io],kn_v[io],ks_v[io]);
-                         }),
+    {
+      io = SubvectorEltIndex(sx_sub, i, j, 0);
+      ke_v[io] = qx_v[io];
+      kw_v[io] = qx_v[io - 1];
+      kn_v[io] = qy_v[io];
+      ks_v[io] = qy_v[io - sy_v];
+      //printf("i=%d j=%d k=%d ke_v=%d kw_v=%d kn_v=%d ks_v=%f\n",i,j,k,ke_v[io],kw_v[io],kn_v[io],ks_v[io]);
+    }),
                          CellFinalize(DoNothing),
                          AfterAllCells(DoNothing)
-      );
+                         );
   }
   else          //fcn = CALCDER calculates the derivs of KE KW KN KS wrt to current cell (i,j,k)
   {
     ForPatchCellsPerFace(BC_ALL,
                          BeforeAllCells(DoNothing),
                          LoopVars(i, j, k, ival, bc_struct, ipatch, sg),
-                         Locals(int io, itop, ip=0, ipp1, ippsy;
+                         Locals(int io, itop, ip = 0, ipp1, ippsy;
                                 int k1, k0x, k0y, k1x, k1y;
-                                double Pupx=NAN, Pupy, Pupox, Pupoy, Pdown, Pdowno;
-                                double Sf_x=NAN, Sf_y, Sf_xo=NAN, Sf_yo=NAN, Sf_mag;),
+                                double Pupx = NAN, Pupy, Pupox, Pupoy, Pdown, Pdowno;
+                                double Sf_x = NAN, Sf_y, Sf_xo = NAN, Sf_yo = NAN, Sf_mag; ),
                          CellSetup(DoNothing),
                          FACE(LeftFace, DoNothing), FACE(RightFace, DoNothing),
                          FACE(DownFace, DoNothing), FACE(UpFace, DoNothing),
                          FACE(BackFace, DoNothing),
                          FACE(FrontFace,
-                         {
-                           io = SubvectorEltIndex(sx_sub, i, j, 0);
-                           itop = SubvectorEltIndex(top_sub, i, j, 0);
-
-                           k1 = (int)top_dat[itop];
-                           k0x = (int)top_dat[itop - 1];
-                           k0y = (int)top_dat[itop - sy_v];
-                           k1x = (int)top_dat[itop + 1];
-                           k1y = (int)top_dat[itop + sy_v];
-
-                           if (k1 >= 0)
-                           {
-                             ip = SubvectorEltIndex(p_sub, i, j, k1);
-                             ipp1 = (int)SubvectorEltIndex(p_sub, i+1, j, k1x);
-                             ippsy = (int)SubvectorEltIndex(p_sub, i, j+1, k1y);
-                             Pupx = pfmax(pp[ipp1], 0.0);
-                             Pupy = pfmax(pp[ippsy], 0.0);
-                             Pupox = pfmax(opp[ipp1], 0.0);
-                             Pupoy = pfmax(opp[ippsy], 0.0);
-                             Pdown = pfmax(pp[ip], 0.0);
-                             Pdowno = pfmax(opp[ip], 0.0);
-
-                             Sf_x = sx_dat[io] + (Pupx - Pdown) / dx;
-                             Sf_y = sy_dat[io] + (Pupy - Pdown) / dy;
-
-                             Sf_xo = sx_dat[io] + (Pupox - Pdowno) / dx;
-                             Sf_yo = sy_dat[io] + (Pupoy - Pdowno) / dy;
-
-                             Sf_mag = RPowerR(Sf_xo * Sf_xo + Sf_yo * Sf_yo, 0.5); //+ov_epsilon;
-                             if (Sf_mag < ov_epsilon)
-                               Sf_mag = ov_epsilon;
-
-                             if (Sf_x < 0)
-                             {
-                               ke_v[io] = (5.0 / 3.0) * (-sx_dat[io] - (Pupx / dx)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io]) * RPowerR(Pdown, (2.0 / 3.0)) +
-                                          (8.0 / 3.0) * RPowerR(Pdown, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dx);
-
-                               kw_v[io + 1] = -RPowerR(Pdown, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dx);
-
-                               ke_vns[io] = kw_v[io + 1];
-                               kw_vns[io + 1] = ke_v[io];
-                             }
-
-                             if (Sf_x >= 0)
-                             {
-                               ke_v[io] = RPowerR(Pupx, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dx);
-
-                               kw_v[io + 1] = (5.0 / 3.0) * (-sx_dat[io] + (Pdown / dx)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io]) * RPowerR(Pupx, (2.0 / 3.0)) -
-                                              (8.0 / 3.0) * RPowerR(Pupx, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dx);
-
-                               ke_vns[io] = kw_v[io + 1];
-                               kw_vns[io + 1] = ke_v[io];
-                             }
-
-                             if (Sf_y < 0)
-                             {
-                               kn_v[io] = (5.0 / 3.0) * (-sy_dat[io] - (Pupy / dy)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io]) * RPowerR(Pdown, (2.0 / 3.0)) +
-                                          (8.0 / 3.0) * RPowerR(Pdown, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dy);
-
-                               ks_v[io + sy_v] = -RPowerR(Pdown, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dy);
-
-                               kn_vns[io] = ks_v[io + sy_v];
-                               ks_vns[io + sy_v] = kn_v[io];
-                             }
-
-                             if (Sf_y >= 0)
-                             {
-                               kn_v[io] = RPowerR(Pupy, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dy);
-
-                               ks_v[io + sy_v] = (5.0 / 3.0) * (-sy_dat[io] + (Pdown / dy)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io]) * RPowerR(Pupy, (2.0 / 3.0)) -
-                                                 (8.0 / 3.0) * RPowerR(Pupy, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dy);
-
-                               kn_vns[io] = ks_v[io + sy_v];
-                               ks_vns[io + sy_v] = kn_v[io];
-                             }
-                           }
-
-                           //fix for lower x boundary
-                           if (k0x < 0.0)
-                           {
-                             Pupx = pfmax((pp[ip]), 0.0);
-                             Sf_x = sx_dat[io] + (Pupx - 0.0) / dx;
-
-                             Pupox = pfmax(opp[ip], 0.0);
-                             Sf_xo = sx_dat[io] + (Pupox - 0.0) / dx;
-
-                             double Sf_mag = RPowerR(Sf_xo * Sf_xo + Sf_yo * Sf_yo, 0.5);
-                             if (Sf_mag < ov_epsilon)
-                               Sf_mag = ov_epsilon;
-
-                             if (Sf_x < 0)
-                             {
-                               ke_v[io - 1] = 0.0;
-                               kw_v[io] = 0.0;
-                               kw_vns[io] = 0.0;
-                               ke_vns[io - 1] = 0.0;
-                             }
-
-                             if (Sf_x >= 0)
-                             {
-                               ke_v[io - 1] = RPowerR(Pupx, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dx);
-                               kw_v[io] = (5.0 / 3.0) * (-sx_dat[io] + 0.0) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io]) * RPowerR(Pupx, (2.0 / 3.0)) -
-                                          (8.0 / 3.0) * RPowerR(Pupx, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dx);
-                               ke_vns[io - 1] = kw_v[io];
-                               kw_vns[io] = ke_v[io - 1];
-                             }
-                           }
-
-                           //fix for lower y boundary
-                           if (k0y < 0.0)
-                           {
-                             Pupy = pfmax((pp[ip]), 0.0);
-                             Sf_y = sy_dat[io] + (Pupy - 0.0) / dy;
-
-                             Pupoy = pfmax(opp[ip], 0.0);
-                             Sf_yo = sy_dat[io] + (Pupoy - 0.0) / dy;
-
-                             double Sf_mag = RPowerR(Sf_xo * Sf_xo + Sf_yo * Sf_yo, 0.5); //Note that the sf_xo was already corrected above
-                             if (Sf_mag < ov_epsilon)
-                               Sf_mag = ov_epsilon;
-
-                             if (Sf_y < 0)
-                             {
-                               kn_v[io - sy_v] = 0.0;
-                               ks_v[io] = 0.0;
-                               ks_vns[io] = 0.0;
-                               kn_vns[io - sy_v] = 0.0;
-                             }
-
-                             if (Sf_y >= 0)
-                             {
-                               kn_vns[io - sy_v] = RPowerR(Pupy, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dy);
-                               ks_v[io] = (5.0 / 3.0) * (-sy_dat[io] + 0.0) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io]) * RPowerR(Pupy, (2.0 / 3.0)) -
-                                          (8.0 / 3.0) * RPowerR(Pupy, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dy);
-                               kn_vns[io - sy_v] = ks_v[io];
-                               ks_vns[io] = kn_v[io - sy_v];
-                             }
-
-                             // Recalculating the x flow in the case with both the lower and left boundaries
-                             // This is exactly the same as the q_x in the left boundary conditional above but
-                             // recalculating qx_v here again becuase the sf_mag will be adjusted with the new sf_yo above
-                             if (k0x < 0.0)
-                             {
-                               if (Sf_x < 0)
-                               {
-                                 kn_v[io - sy_v] = 0.0;
-                                 ks_v[io] = 0.0;
-                                 ks_vns[io] = 0.0;
-                                 kn_vns[io - sy_v] = 0.0;
-                               }
-
-                               if (Sf_x >= 0)
-                               {
-                                 ke_v[io - 1] = RPowerR(Pupx, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dx);
-                                 kw_v[io] = (5.0 / 3.0) * (-sx_dat[io] + 0.0) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io]) * RPowerR(Pupx, (2.0 / 3.0)) -
-                                            (8.0 / 3.0) * RPowerR(Pupx, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dx);
-                                 ke_vns[io - 1] = kw_v[io];
-                                 kw_vns[io] = ke_v[io - 1];
-                               }
-                             }
-                           }
-                         }),
+    {
+      io = SubvectorEltIndex(sx_sub, i, j, 0);
+      itop = SubvectorEltIndex(top_sub, i, j, 0);
+
+      k1 = (int)top_dat[itop];
+      k0x = (int)top_dat[itop - 1];
+      k0y = (int)top_dat[itop - sy_v];
+      k1x = (int)top_dat[itop + 1];
+      k1y = (int)top_dat[itop + sy_v];
+
+      if (k1 >= 0)
+      {
+        ip = SubvectorEltIndex(p_sub, i, j, k1);
+        ipp1 = (int)SubvectorEltIndex(p_sub, i + 1, j, k1x);
+        ippsy = (int)SubvectorEltIndex(p_sub, i, j + 1, k1y);
+        Pupx = pfmax(pp[ipp1], 0.0);
+        Pupy = pfmax(pp[ippsy], 0.0);
+        Pupox = pfmax(opp[ipp1], 0.0);
+        Pupoy = pfmax(opp[ippsy], 0.0);
+        Pdown = pfmax(pp[ip], 0.0);
+        Pdowno = pfmax(opp[ip], 0.0);
+
+        Sf_x = sx_dat[io] + (Pupx - Pdown) / dx;
+        Sf_y = sy_dat[io] + (Pupy - Pdown) / dy;
+
+        Sf_xo = sx_dat[io] + (Pupox - Pdowno) / dx;
+        Sf_yo = sy_dat[io] + (Pupoy - Pdowno) / dy;
+
+        Sf_mag = RPowerR(Sf_xo * Sf_xo + Sf_yo * Sf_yo, 0.5);                      //+ov_epsilon;
+        if (Sf_mag < ov_epsilon)
+          Sf_mag = ov_epsilon;
+
+        if (Sf_x < 0)
+        {
+          ke_v[io] = (5.0 / 3.0) * (-sx_dat[io] - (Pupx / dx)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io]) * RPowerR(Pdown, (2.0 / 3.0)) +
+                     (8.0 / 3.0) * RPowerR(Pdown, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dx);
+
+          kw_v[io + 1] = -RPowerR(Pdown, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dx);
+
+          ke_vns[io] = kw_v[io + 1];
+          kw_vns[io + 1] = ke_v[io];
+        }
+
+        if (Sf_x >= 0)
+        {
+          ke_v[io] = RPowerR(Pupx, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dx);
+
+          kw_v[io + 1] = (5.0 / 3.0) * (-sx_dat[io] + (Pdown / dx)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io]) * RPowerR(Pupx, (2.0 / 3.0)) -
+                         (8.0 / 3.0) * RPowerR(Pupx, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dx);
+
+          ke_vns[io] = kw_v[io + 1];
+          kw_vns[io + 1] = ke_v[io];
+        }
+
+        if (Sf_y < 0)
+        {
+          kn_v[io] = (5.0 / 3.0) * (-sy_dat[io] - (Pupy / dy)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io]) * RPowerR(Pdown, (2.0 / 3.0)) +
+                     (8.0 / 3.0) * RPowerR(Pdown, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dy);
+
+          ks_v[io + sy_v] = -RPowerR(Pdown, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dy);
+
+          kn_vns[io] = ks_v[io + sy_v];
+          ks_vns[io + sy_v] = kn_v[io];
+        }
+
+        if (Sf_y >= 0)
+        {
+          kn_v[io] = RPowerR(Pupy, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dy);
+
+          ks_v[io + sy_v] = (5.0 / 3.0) * (-sy_dat[io] + (Pdown / dy)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io]) * RPowerR(Pupy, (2.0 / 3.0)) -
+                            (8.0 / 3.0) * RPowerR(Pupy, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dy);
+
+          kn_vns[io] = ks_v[io + sy_v];
+          ks_vns[io + sy_v] = kn_v[io];
+        }
+      }
+
+      //fix for lower x boundary
+      if (k0x < 0.0)
+      {
+        Pupx = pfmax((pp[ip]), 0.0);
+        Sf_x = sx_dat[io] + (Pupx - 0.0) / dx;
+
+        Pupox = pfmax(opp[ip], 0.0);
+        Sf_xo = sx_dat[io] + (Pupox - 0.0) / dx;
+
+        double Sf_mag = RPowerR(Sf_xo * Sf_xo + Sf_yo * Sf_yo, 0.5);
+        if (Sf_mag < ov_epsilon)
+          Sf_mag = ov_epsilon;
+
+        if (Sf_x < 0)
+        {
+          ke_v[io - 1] = 0.0;
+          kw_v[io] = 0.0;
+          kw_vns[io] = 0.0;
+          ke_vns[io - 1] = 0.0;
+        }
+
+        if (Sf_x >= 0)
+        {
+          ke_v[io - 1] = RPowerR(Pupx, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dx);
+          kw_v[io] = (5.0 / 3.0) * (-sx_dat[io] + 0.0) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io]) * RPowerR(Pupx, (2.0 / 3.0)) -
+                     (8.0 / 3.0) * RPowerR(Pupx, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dx);
+          ke_vns[io - 1] = kw_v[io];
+          kw_vns[io] = ke_v[io - 1];
+        }
+      }
+
+      //fix for lower y boundary
+      if (k0y < 0.0)
+      {
+        Pupy = pfmax((pp[ip]), 0.0);
+        Sf_y = sy_dat[io] + (Pupy - 0.0) / dy;
+
+        Pupoy = pfmax(opp[ip], 0.0);
+        Sf_yo = sy_dat[io] + (Pupoy - 0.0) / dy;
+
+        double Sf_mag = RPowerR(Sf_xo * Sf_xo + Sf_yo * Sf_yo, 0.5);                      //Note that the sf_xo was already corrected above
+        if (Sf_mag < ov_epsilon)
+          Sf_mag = ov_epsilon;
+
+        if (Sf_y < 0)
+        {
+          kn_v[io - sy_v] = 0.0;
+          ks_v[io] = 0.0;
+          ks_vns[io] = 0.0;
+          kn_vns[io - sy_v] = 0.0;
+        }
+
+        if (Sf_y >= 0)
+        {
+          kn_vns[io - sy_v] = RPowerR(Pupy, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dy);
+          ks_v[io] = (5.0 / 3.0) * (-sy_dat[io] + 0.0) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io]) * RPowerR(Pupy, (2.0 / 3.0)) -
+                     (8.0 / 3.0) * RPowerR(Pupy, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dy);
+          kn_vns[io - sy_v] = ks_v[io];
+          ks_vns[io] = kn_v[io - sy_v];
+        }
+
+        // Recalculating the x flow in the case with both the lower and left boundaries
+        // This is exactly the same as the q_x in the left boundary conditional above but
+        // recalculating qx_v here again becuase the sf_mag will be adjusted with the new sf_yo above
+        if (k0x < 0.0)
+        {
+          if (Sf_x < 0)
+          {
+            kn_v[io - sy_v] = 0.0;
+            ks_v[io] = 0.0;
+            ks_vns[io] = 0.0;
+            kn_vns[io - sy_v] = 0.0;
+          }
+
+          if (Sf_x >= 0)
+          {
+            ke_v[io - 1] = RPowerR(Pupx, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dx);
+            kw_v[io] = (5.0 / 3.0) * (-sx_dat[io] + 0.0) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io]) * RPowerR(Pupx, (2.0 / 3.0)) -
+                       (8.0 / 3.0) * RPowerR(Pupx, (5.0 / 3.0)) / (RPowerR(fabs(Sf_mag), 0.5) * mann_dat[io] * dx);
+            ke_vns[io - 1] = kw_v[io];
+            kw_vns[io] = ke_v[io - 1];
+          }
+        }
+      }
+    }),
                          CellFinalize(DoNothing),
                          AfterAllCells(DoNothing)
-      );
+                         );
   }
 }
 
diff --git a/pfsimulator/parflow_lib/overlandsum.c b/pfsimulator/parflow_lib/overlandsum.c
index 09219cebf..eccd6fa4e 100644
--- a/pfsimulator/parflow_lib/overlandsum.c
+++ b/pfsimulator/parflow_lib/overlandsum.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
diff --git a/pfsimulator/parflow_lib/parflow.h b/pfsimulator/parflow_lib/parflow.h
index 171cd1df5..0c309fed8 100644
--- a/pfsimulator/parflow_lib/parflow.h
+++ b/pfsimulator/parflow_lib/parflow.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 * Header file to include all header information for parflow.
@@ -66,6 +66,7 @@
 #include "problem_bc.h"
 #include "problem_eval.h"
 #include "well.h"
+#include "reservoir.h"
 #include "bc_pressure.h"
 #include "problem.h"
 #include "solver.h"
diff --git a/pfsimulator/parflow_lib/parflow_netcdf.h b/pfsimulator/parflow_lib/parflow_netcdf.h
index 1b4daabf5..6efb9d8fa 100644
--- a/pfsimulator/parflow_lib/parflow_netcdf.h
+++ b/pfsimulator/parflow_lib/parflow_netcdf.h
@@ -1,23 +1,30 @@
-/*BHEADER*********************************************************************
- *  This file is part of Parflow. For details, see
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifdef PARFLOW_HAVE_NETCDF
 #include <netcdf.h>
@@ -49,7 +56,7 @@ void PutDataInNCNode(int varID, double *data_nc_node, int *nodeXIndices, int *no
                      int *nodeXCount, int *nodeYCount, int *nodeZCount, double t, varNCData *myVarNCData, int *netCDFIDs);
 void ReadPFNC(char *fileName, Vector *v, char *varName, int tStep, int dimensionality);
 void OpenNCFile(char *file_name, int *ncRID);
-void ReadNCFile(int ncRID, int varID, Subvector *subvector, Subgrid *subgrid, char *varName, int tStep, int dimensionality);
+void ReadNCFile(char* filename, int ncRID, Subvector *subvector, Subgrid *subgrid, char *varName, int tStep, int dimensionality);
 
 
 
diff --git a/pfsimulator/parflow_lib/parflow_proto.h b/pfsimulator/parflow_lib/parflow_proto.h
index fad5caa5e..c8165cdd0 100644
--- a/pfsimulator/parflow_lib/parflow_proto.h
+++ b/pfsimulator/parflow_lib/parflow_proto.h
@@ -81,7 +81,7 @@ int ChebyshevSizeOfTempData(void);
 
 /* comm_pkg.c */
 void ProjectRegion(Region *region, int sx, int sy, int sz, int ix, int iy, int iz);
-Region *ProjectRBPoint(Region *region, int rb [4 ][3 ]);
+Region *ProjectRBPoint(Region *region, int rb[4 ][3 ]);
 void CreateComputePkgs(Grid *grid);
 void FreeComputePkgs(Grid *grid);
 
@@ -481,7 +481,7 @@ void PerturbSystem(Lattice *lattice, Problem *problem);
 
 /* pf_module.c */
 PFModule *NewPFModule(void *call, void *init_instance_xtra, void *free_instance_xtra, void *new_public_xtra, void *free_public_xtra, void *sizeof_temp_data, void *instance_xtra, void *public_xtra);
-PFModule *NewPFModuleExtended(void *call, void *init_instance_xtra, void *free_instance_xtra, void *new_public_xtra, void *free_public_xtra, void *sizeof_temp_data,  void *output, void *output_static,void *instance_xtra, void *public_xtra);
+PFModule *NewPFModuleExtended(void *call, void *init_instance_xtra, void *free_instance_xtra, void *new_public_xtra, void *free_public_xtra, void *sizeof_temp_data, void *output, void *output_static, void *instance_xtra, void *public_xtra);
 PFModule *DupPFModule(PFModule *pf_module);
 void FreePFModule(PFModule *pf_module);
 
@@ -515,8 +515,8 @@ PFModule  *SMGInitInstanceXtra(
                                Problem *    problem,
                                Grid *       grid,
                                ProblemData *problem_data,
-			       Matrix *     pf_Bmat,
-			       Matrix *     pf_Cmat,
+                               Matrix *     pf_Bmat,
+                               Matrix *     pf_Cmat,
                                double *     temp_data);
 void SMGFreeInstanceXtra(void);
 PFModule *SMGNewPublicXtra(char *name);
@@ -1009,6 +1009,24 @@ PFModule *YSlopeNewPublicXtra(void);
 void YSlopeFreePublicXtra(void);
 int YSlopeSizeOfTempData(void);
 
+typedef void (*ChannelWidthInvoke) (ProblemData *problem_data, Vector *wc_x, Vector *dummy);
+typedef PFModule *(*ChannelWidthInitInstanceXtraInvoke) (Grid *grid3d, Grid *grid2d);
+/* problem_wc_x.c */
+void XChannelWidth(ProblemData *problem_data, Vector *x_wc, Vector *dummy);
+PFModule *XChannelWidthInitInstanceXtra(Grid *grid3d, Grid *grid2d);
+void XChannelWidthFreeInstanceXtra(void);
+PFModule *XChannelWidthNewPublicXtra(void);
+void XChannelWidthFreePublicXtra(void);
+int XChannelWidthSizeOfTempData(void);
+
+/* problem_wc_y.c */
+void YChannelWidth(ProblemData *problem_data, Vector *wc_y, Vector *dummy);
+PFModule *YChannelWidthInitInstanceXtra(Grid *grid3d, Grid *grid2d);
+void YChannelWidthFreeInstanceXtra(void);
+PFModule *YChannelWidthNewPublicXtra(void);
+void YChannelWidthFreePublicXtra(void);
+int YChannelWidthSizeOfTempData(void);
+
 /* random.c */
 void SeedRand(int seed);
 double Rand(void);
@@ -1303,9 +1321,28 @@ PFModule *WellPackageNewPublicXtra(int num_phases, int num_contaminants);
 void WellPackageFreePublicXtra(void);
 int WellPackageSizeOfTempData(void);
 
+/* reservoir.c */
+ReservoirData *NewReservoirData(void);
+void FreeReservoirData(ReservoirData *reservoir_data);
+void PrintReservoirData(ReservoirData *reservoir_data, unsigned int print_mask);
+void WriteReservoirs(char *file_prefix, Problem *problem, ReservoirData *reservoir_data, double time, int write_header);
+
+typedef void (*ReservoirPackageInvoke) (ProblemData *problem_data);
+typedef PFModule *(*ReservoirPackageNewPublicXtraInvoke) (int num_phases, int num_contaminants);
+
+
+
 /* wells_lb.c */
 void LBWells(Lattice *lattice, Problem *problem, ProblemData *problem_data);
 
+/* reservoir_package.c */
+void ReservoirPackage(ProblemData *problem_data);
+PFModule *ReservoirPackageInitInstanceXtra(void);
+void ReservoirPackageFreeInstanceXtra(void);
+PFModule *ReservoirPackageNewPublicXtra(void);
+void ReservoirPackageFreePublicXtra(void);
+int ReservoirPackageSizeOfTempData(void);
+
 /* write_parflow_binary.c */
 long SizeofPFBinarySubvector(Subvector *subvector, Subgrid *subgrid);
 void WritePFBinary_Subvector(amps_File file, Subvector *subvector, Subgrid *subgrid);
@@ -1372,7 +1409,7 @@ void ComputeTop(Problem *    problem,
                 ProblemData *problem_data);
 
 void ComputePatchTop(Problem *    problem,
-		     ProblemData *problem_data);
+                     ProblemData *problem_data);
 
 int CheckTime(Problem *problem, char *key, double time);
 
diff --git a/pfsimulator/parflow_lib/parflow_proto_f.h b/pfsimulator/parflow_lib/parflow_proto_f.h
index 8ae045a2c..819bf583c 100644
--- a/pfsimulator/parflow_lib/parflow_proto_f.h
+++ b/pfsimulator/parflow_lib/parflow_proto_f.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- * Copyright (c) 1995-2009, Lawrence Livermore National Security,
- * LLC. Produced at the Lawrence Livermore National Laboratory. Written
- * by the Parflow Team (see the CONTRIBUTORS file)
- * <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- * This file is part of Parflow. For details, see
- * http://www.llnl.gov/casc/parflow
- *
- * Please read the COPYRIGHT file or Our Notice and the LICENSE file
- * for the GNU Lesser General Public License.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License (as published
- * by the Free Software Foundation) version 2.1 dated February 1999.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- * and conditions of the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- * USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 * C to Fortran interfacing macros
@@ -56,16 +56,16 @@ BEGIN_EXTERN_C
 #define ADVECT advect_
 #endif
 
-#define CALL_ADVECT(s, sn, uedge, vedge, wedge, phi,                      \
-                    slx, sly, slz,                                        \
-                    lo, hi, dlo, dhi, hx, dt, fstord,                     \
-                    sbot, stop, sbotp, sfrt, sbck, sleft, sright, sfluxz, \
-                    dxscr, dyscr, dzscr, dzfrm)                           \
-  ADVECT(s, sn, uedge, vedge, wedge, phi,                                 \
-         slx, sly, slz,                                                   \
-         lo, hi, dlo, dhi, hx, &dt, &fstord,                              \
-         sbot, stop, sbotp, sfrt, sbck, sleft, sright, sfluxz,            \
-         dxscr, dyscr, dzscr, dzfrm)
+#define CALL_ADVECT(s, sn, uedge, vedge, wedge, phi,                            \
+                    slx, sly, slz,                                              \
+                    lo, hi, dlo, dhi, hx, dt, fstord,                           \
+                    sbot, stop, sbotp, sfrt, sbck, sleft, sright, sfluxz,       \
+                    dxscr, dyscr, dzscr, dzfrm)                                 \
+        ADVECT(s, sn, uedge, vedge, wedge, phi,                                 \
+               slx, sly, slz,                                                   \
+               lo, hi, dlo, dhi, hx, &dt, &fstord,                              \
+               sbot, stop, sbotp, sfrt, sbck, sleft, sright, sfluxz,            \
+               dxscr, dyscr, dzscr, dzfrm)
 
 void ADVECT(double *s, double *sn,
             double *uedge, double *vedge, double *wedge, double *phi,
@@ -85,18 +85,18 @@ void ADVECT(double *s, double *sn,
 #define SADVECT sadvect_
 #endif
 
-#define CALL_SADVECT(s, sn, uedge, vedge, wedge, betaedge, phi,            \
-                     viscosity, density, gravity,                          \
-                     slx, sly, slz,                                        \
-                     lohi, dlohi, hx, dt,                                  \
-                     sbot, stop, sbotp, sfrt, sbck, sleft, sright, sfluxz, \
-                     dxscr, dyscr, dzscr, dzfrm)                           \
-  SADVECT(s, sn, uedge, vedge, wedge, betaedge, phi,                       \
-          viscosity, density, &gravity,                                    \
-          slx, sly, slz,                                                   \
-          lohi, dlohi, hx, &dt,                                            \
-          sbot, stop, sbotp, sfrt, sbck, sleft, sright, sfluxz,            \
-          dxscr, dyscr, dzscr, dzfrm)
+#define CALL_SADVECT(s, sn, uedge, vedge, wedge, betaedge, phi,                  \
+                     viscosity, density, gravity,                                \
+                     slx, sly, slz,                                              \
+                     lohi, dlohi, hx, dt,                                        \
+                     sbot, stop, sbotp, sfrt, sbck, sleft, sright, sfluxz,       \
+                     dxscr, dyscr, dzscr, dzfrm)                                 \
+        SADVECT(s, sn, uedge, vedge, wedge, betaedge, phi,                       \
+                viscosity, density, &gravity,                                    \
+                slx, sly, slz,                                                   \
+                lohi, dlohi, hx, &dt,                                            \
+                sbot, stop, sbotp, sfrt, sbck, sleft, sright, sfluxz,            \
+                dxscr, dyscr, dzscr, dzfrm)
 
 void SADVECT(double *s, double *sn,
              double *uedge, double *vedge, double *wedge, double *betaedge, double *phi,
@@ -120,31 +120,31 @@ void SADVECT(double *s, double *sn,
 #define CLM_LSM clm_lsm_
 #endif
 
-#define CALL_CLM_LSM(pressure_data, saturation_data, evap_trans_data, mask, porosity_data,                                                                                     \
-                     dz_mult_data, istep, dt, t, start_time, dx, dy, dz, ix, iy, nx, ny, nz,                                                                                   \
-                     nx_f, ny_f, nz_f, nz_rz, ip, p, q, r, gnx, gny, rank,                                                                                                     \
-                     sw_data, lw_data, prcp_data, tas_data, u_data, v_data, patm_data, qatm_data,                                                                              \
-                     lai_data, sai_data, z0m_data, displa_data,                                                                                                                \
-                     slope_x_data, slope_y_data,                                                                                                                               \
-                     eflx_lh_tot_data, eflx_lwrad_out_data, eflx_sh_tot_data, eflx_soil_grnd_data,                                                                             \
-                     qflx_evap_tot_data, qflx_evap_grnd_data, qflx_evap_soi_data, qflx_evap_veg_data, qflx_tran_veg_data,                                                      \
-                     qflx_infl_data, swe_out_data, t_grnd_data, t_soil_data,                                                                                                   \
-                     clm_dump_interval, clm_1d_out, clm_forc_veg, clm_file_dir, clm_file_dir_length, clm_bin_out_dir, write_CLM_binary, slope_accounting_CLM,                                       \
-                     clm_beta_function, clm_veg_function, clm_veg_wilting, clm_veg_fieldc, clm_res_sat,                                                                        \
-                     clm_irr_type, clm_irr_cycle, clm_irr_rate, clm_irr_start, clm_irr_stop,                                                                                   \
-                     clm_irr_threshold, qirr, qirr_inst, iflag, clm_irr_thresholdtype, soi_z, clm_next, clm_write_logs, clm_last_rst, clm_daily_rst, clm_nlevsoi, clm_nlevlak) \
-  CLM_LSM(pressure_data, saturation_data, evap_trans_data, mask, porosity_data,                                                                                                \
-          dz_mult_data, &istep, &dt, &t, &start_time, &dx, &dy, &dz, &ix, &iy, &nx, &ny, &nz, &nx_f, &ny_f, &nz_f, &nz_rz, &ip, &p, &q, &r, &gnx, &gny, &rank,                 \
-          sw_data, lw_data, prcp_data, tas_data, u_data, v_data, patm_data, qatm_data,                                                                                         \
-          lai_data, sai_data, z0m_data, displa_data,                                                                                                                           \
-          slope_x_data, slope_y_data,                                                                                                                                          \
-          eflx_lh_tot_data, eflx_lwrad_out_data, eflx_sh_tot_data, eflx_soil_grnd_data,                                                                                        \
-          qflx_evap_tot_data, qflx_evap_grnd_data, qflx_evap_soi_data, qflx_evap_veg_data, qflx_tran_veg_data,                                                                 \
-          qflx_infl_data, swe_out_data, t_grnd_data, t_soil_data,                                                                                                              \
-          &clm_dump_interval, &clm_1d_out, &clm_forc_veg, clm_file_dir, &clm_file_dir_length, &clm_bin_out_dir,                                                                \
-          &write_CLM_binary, &slope_accounting_CLM, &clm_beta_function, &clm_veg_function, &clm_veg_wilting, &clm_veg_fieldc,                                                                         \
-          &clm_res_sat, &clm_irr_type, &clm_irr_cycle, &clm_irr_rate, &clm_irr_start, &clm_irr_stop,                                                                           \
-          &clm_irr_threshold, qirr, qirr_inst, iflag, &clm_irr_thresholdtype, &soi_z, &clm_next, &clm_write_logs, &clm_last_rst, &clm_daily_rst, &clm_nlevsoi, &clm_nlevlak);
+#define CALL_CLM_LSM(pressure_data, saturation_data, evap_trans_data, mask, porosity_data,                                                                                                                  \
+                     dz_mult_data, istep, dt, t, start_time, dx, dy, dz, ix, iy, nx, ny, nz,                                                                                                                \
+                     nx_f, ny_f, nz_f, nz_rz, ip, p, q, r, gnx, gny, rank,                                                                                                                                  \
+                     sw_data, lw_data, prcp_data, tas_data, u_data, v_data, patm_data, qatm_data,                                                                                                           \
+                     lai_data, sai_data, z0m_data, displa_data,                                                                                                                                             \
+                     slope_x_data, slope_y_data,                                                                                                                                                            \
+                     eflx_lh_tot_data, eflx_lwrad_out_data, eflx_sh_tot_data, eflx_soil_grnd_data,                                                                                                          \
+                     qflx_evap_tot_data, qflx_evap_grnd_data, qflx_evap_soi_data, qflx_evap_veg_data, qflx_tran_veg_data,                                                                                   \
+                     qflx_infl_data, swe_out_data, t_grnd_data, t_soil_data,                                                                                                                                \
+                     clm_dump_interval, clm_1d_out, clm_forc_veg, clm_file_dir, clm_file_dir_length, clm_bin_out_dir, write_CLM_binary, slope_accounting_CLM,                                               \
+                     clm_beta_function, clm_veg_function, clm_veg_wilting, clm_veg_fieldc, clm_res_sat,                                                                                                     \
+                     clm_irr_type, clm_irr_cycle, clm_irr_rate, clm_irr_start, clm_irr_stop,                                                                                                                \
+                     clm_irr_threshold, qirr, qirr_inst, iflag, clm_irr_thresholdtype, soi_z, clm_next, clm_write_logs, clm_last_rst, clm_daily_rst, clm_nlevsoi, clm_nlevlak)                              \
+        CLM_LSM(pressure_data, saturation_data, evap_trans_data, mask, porosity_data,                                                                                                                       \
+                dz_mult_data, &istep, &dt, &t, &start_time, &dx, &dy, &dz, &ix, &iy, &nx, &ny, &nz, &nx_f, &ny_f, &nz_f, &nz_rz, &ip, &p, &q, &r, &gnx, &gny, &rank,                                        \
+                sw_data, lw_data, prcp_data, tas_data, u_data, v_data, patm_data, qatm_data,                                                                                                                \
+                lai_data, sai_data, z0m_data, displa_data,                                                                                                                                                  \
+                slope_x_data, slope_y_data,                                                                                                                                                                 \
+                eflx_lh_tot_data, eflx_lwrad_out_data, eflx_sh_tot_data, eflx_soil_grnd_data,                                                                                                               \
+                qflx_evap_tot_data, qflx_evap_grnd_data, qflx_evap_soi_data, qflx_evap_veg_data, qflx_tran_veg_data,                                                                                        \
+                qflx_infl_data, swe_out_data, t_grnd_data, t_soil_data,                                                                                                                                     \
+                &clm_dump_interval, &clm_1d_out, &clm_forc_veg, clm_file_dir, &clm_file_dir_length, &clm_bin_out_dir,                                                                                       \
+                &write_CLM_binary, &slope_accounting_CLM, &clm_beta_function, &clm_veg_function, &clm_veg_wilting, &clm_veg_fieldc,                                                                         \
+                &clm_res_sat, &clm_irr_type, &clm_irr_cycle, &clm_irr_rate, &clm_irr_start, &clm_irr_stop,                                                                                                  \
+                &clm_irr_threshold, qirr, qirr_inst, iflag, &clm_irr_thresholdtype, &soi_z, &clm_next, &clm_write_logs, &clm_last_rst, &clm_daily_rst, &clm_nlevsoi, &clm_nlevlak);
 
 void CLM_LSM(double *pressure_data, double *saturation_data, double *evap_trans_data, double *mask, double *porosity_data,
              double *dz_mult_data, int *istep, double *dt, double *t, double *start_time,
diff --git a/pfsimulator/parflow_lib/parflow_proto_f90.h b/pfsimulator/parflow_lib/parflow_proto_f90.h
index a6f1626d4..e64018349 100644
--- a/pfsimulator/parflow_lib/parflow_proto_f90.h
+++ b/pfsimulator/parflow_lib/parflow_proto_f90.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 * C to Fortran interfacing macros
@@ -40,16 +40,16 @@
 #define ADVECT advect_
 #endif
 
-#define CALL_ADVECT(s, sn, uedge, vedge, wedge, phi,                      \
-                    slx, sly, slz,                                        \
-                    lo, hi, dlo, dhi, hx, dt, fstord,                     \
-                    sbot, stop, sbotp, sfrt, sbck, sleft, sright, sfluxz, \
-                    dxscr, dyscr, dzscr, dzfrm)                           \
-  ADVECT(s, sn, uedge, vedge, wedge, phi,                                 \
-         slx, sly, slz,                                                   \
-         lo, hi, dlo, dhi, hx, &dt, &fstord,                              \
-         sbot, stop, sbotp, sfrt, sbck, sleft, sright, sfluxz,            \
-         dxscr, dyscr, dzscr, dzfrm)
+#define CALL_ADVECT(s, sn, uedge, vedge, wedge, phi,                            \
+                    slx, sly, slz,                                              \
+                    lo, hi, dlo, dhi, hx, dt, fstord,                           \
+                    sbot, stop, sbotp, sfrt, sbck, sleft, sright, sfluxz,       \
+                    dxscr, dyscr, dzscr, dzfrm)                                 \
+        ADVECT(s, sn, uedge, vedge, wedge, phi,                                 \
+               slx, sly, slz,                                                   \
+               lo, hi, dlo, dhi, hx, &dt, &fstord,                              \
+               sbot, stop, sbotp, sfrt, sbck, sleft, sright, sfluxz,            \
+               dxscr, dyscr, dzscr, dzfrm)
 
 void ADVECT(double *s, double *sn,
             double *uedge, double *vedge, double *wedge, double *phi,
@@ -69,18 +69,18 @@ void ADVECT(double *s, double *sn,
 #define SADVECT sadvect_
 #endif
 
-#define CALL_SADVECT(s, sn, uedge, vedge, wedge, betaedge, phi,            \
-                     viscosity, density, gravity,                          \
-                     slx, sly, slz,                                        \
-                     lohi, dlohi, hx, dt,                                  \
-                     sbot, stop, sbotp, sfrt, sbck, sleft, sright, sfluxz, \
-                     dxscr, dyscr, dzscr, dzfrm)                           \
-  SADVECT(s, sn, uedge, vedge, wedge, betaedge, phi,                       \
-          viscosity, density, &gravity,                                    \
-          slx, sly, slz,                                                   \
-          lohi, dlohi, hx, &dt,                                            \
-          sbot, stop, sbotp, sfrt, sbck, sleft, sright, sfluxz,            \
-          dxscr, dyscr, dzscr, dzfrm)
+#define CALL_SADVECT(s, sn, uedge, vedge, wedge, betaedge, phi,                  \
+                     viscosity, density, gravity,                                \
+                     slx, sly, slz,                                              \
+                     lohi, dlohi, hx, dt,                                        \
+                     sbot, stop, sbotp, sfrt, sbck, sleft, sright, sfluxz,       \
+                     dxscr, dyscr, dzscr, dzfrm)                                 \
+        SADVECT(s, sn, uedge, vedge, wedge, betaedge, phi,                       \
+                viscosity, density, &gravity,                                    \
+                slx, sly, slz,                                                   \
+                lohi, dlohi, hx, &dt,                                            \
+                sbot, stop, sbotp, sfrt, sbck, sleft, sright, sfluxz,            \
+                dxscr, dyscr, dzscr, dzfrm)
 
 void SADVECT(double *s, double *sn,
              double *uedge, double *vedge, double *wedge, double *betaedge, double *phi,
@@ -96,6 +96,6 @@ void SADVECT(double *s, double *sn,
 #define FTEST ftest_
 
 #define CALL_FTEST(outflow_log) \
-  FTEST(outflow_log);
+        FTEST(outflow_log);
 
 void FTEST(double *outflow_log);
diff --git a/pfsimulator/parflow_lib/pcg.c b/pfsimulator/parflow_lib/pcg.c
index 224e4feac..891fa4443 100644
--- a/pfsimulator/parflow_lib/pcg.c
+++ b/pfsimulator/parflow_lib/pcg.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Preconditioned conjugate gradient solver (Omin).
@@ -449,7 +449,7 @@ PFModule   *PCGNewPublicXtra(char *name)
 
     default:
     {
-      InputError("Invalid switch value <%s> for key <%s>", switch_name, key);	    
+      InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
     }
   }
 
diff --git a/pfsimulator/parflow_lib/permeability_face.c b/pfsimulator/parflow_lib/permeability_face.c
index 28294d313..c41f1bd04 100644
--- a/pfsimulator/parflow_lib/permeability_face.c
+++ b/pfsimulator/parflow_lib/permeability_face.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *****************************************************************************/
@@ -237,7 +237,7 @@ void PermeabilityFaceFreePublicXtra()
 
   if (public_xtra)
   {
-    free(public_xtra);
+    tfree(public_xtra);
   }
 }
 
diff --git a/pfsimulator/parflow_lib/pf_cudaloops.h b/pfsimulator/parflow_lib/pf_cudaloops.h
index a68a6566f..f57e2600a 100644
--- a/pfsimulator/parflow_lib/pf_cudaloops.h
+++ b/pfsimulator/parflow_lib/pf_cudaloops.h
@@ -30,52 +30,51 @@
 #include "pf_devices.h"
 #include "pf_cudamalloc.h"
 
-extern "C++"{
-
+extern "C++" {
 #include <tuple>
-#include "cub.cuh"
+#include <cub/cub.cuh>
 
 /*--------------------------------------------------------------------------
  * CUDA blocksize definitions
  *--------------------------------------------------------------------------*/
 
 /**
- * The largest blocksize ParFlow is using, but also the largest blocksize 
- * supported by any currently available NVIDIA GPU architecture. This can 
- * also differ between different architectures. It is used for informing 
- * the compiler about how many registers should be available for the GPU 
- * kernel during the compilation. Another option is to use 
- * --maxrregcount 64 compiler flag, but NVIDIA recommends specifying 
+ * The largest blocksize ParFlow is using, but also the largest blocksize
+ * supported by any currently available NVIDIA GPU architecture. This can
+ * also differ between different architectures. It is used for informing
+ * the compiler about how many registers should be available for the GPU
+ * kernel during the compilation. Another option is to use
+ * --maxrregcount 64 compiler flag, but NVIDIA recommends specifying
  * this kernel-by-kernel basis by __launch_bounds__() identifier.
  */
 #define BLOCKSIZE_MAX 1024
 
 /**
- * The blocksize for the x-dimension. This is is set to 32, 
- * because the warp size for the current NVIDIA architectures is 32. 
- * Therefore, letting each thread in a warp access consecutive memory 
- * locations along the x-dimension results in best memory coalescence. 
- * It is also important that the total blocksize (the product of x, y, 
+ * The blocksize for the x-dimension. This is is set to 32,
+ * because the warp size for the current NVIDIA architectures is 32.
+ * Therefore, letting each thread in a warp access consecutive memory
+ * locations along the x-dimension results in best memory coalescence.
+ * It is also important that the total blocksize (the product of x, y,
  * and z-blocksizes) is divisible by the warp size (32).
  */
 #define BLOCKSIZE_X 32
 
 /**
- * The default blocksize for the y-dimension. Blocksizes along y and 
- * z-dimensions are less important compared to the x-dimension. 
+ * The default blocksize for the y-dimension. Blocksizes along y and
+ * z-dimensions are less important compared to the x-dimension.
  */
 #define BLOCKSIZE_Y 8
 
 /**
- * The default blocksize for the z-dimension. Blocksizes along y and 
- * z-dimensions are less important compared to the x-dimension. 
+ * The default blocksize for the z-dimension. Blocksizes along y and
+ * z-dimensions are less important compared to the x-dimension.
  */
 #define BLOCKSIZE_Z 4
 
 /*--------------------------------------------------------------------------
  * CUDA lambda definition (visible for host and device functions)
  *--------------------------------------------------------------------------*/
-#define GPU_LAMBDA [=] __host__ __device__
+#define GPU_LAMBDA [ = ] __host__ __device__
 
 /*--------------------------------------------------------------------------
  * CUDA helper macros and functions
@@ -88,37 +87,36 @@ extern "C++"{
 #define RAND48_MULT_2   (0x0005)
 #define RAND48_ADD      (0x000b)
 
- /** Helper struct for type comparison. @note Not for direct use! */
-template <typename T>
+/** Helper struct for type comparison. @note Not for direct use! */
+template < typename T >
 struct function_traits
-    : public function_traits<decltype(&T::operator())>
+  : public function_traits < decltype(&T::operator()) >
 {};
 // For generic types, directly use the result of the signature of its 'operator()'
 
- /** Helper struct for type comparison. @note Not for direct use! */
-template <typename ClassType, typename ReturnType, typename... Args>
-struct function_traits<ReturnType(ClassType::*)(Args...) const>
+/** Helper struct for type comparison. @note Not for direct use! */
+template < typename ClassType, typename ReturnType, typename ... Args >
+struct function_traits < ReturnType (ClassType::*)(Args...) const >
 // we specialize for pointers to member function
 {
-    enum { arity = sizeof...(Args) };
-    // arity is the number of arguments.
+  enum { arity = sizeof...(Args) };
+  // arity is the number of arguments.
 
-    typedef ReturnType result_type;
+  typedef ReturnType result_type;
 
-    template <size_t i>
-    struct arg
-    {
-        typedef typename std::tuple_element<i, std::tuple<Args...>>::type type;
-        // the i-th argument is equivalent to the i-th tuple element of a tuple
-        // composed of those arguments.
-    };
+  template < size_t i >
+  struct arg {
+    typedef typename std::tuple_element < i, std::tuple < Args... > > ::type type;
+    // the i-th argument is equivalent to the i-th tuple element of a tuple
+    // composed of those arguments.
+  };
 };
 
 /** Device-callable dorand48() function for CUDA compute kernels. */
 __host__ __device__ __forceinline__ static void dev_dorand48(unsigned short xseed[3])
 {
   unsigned long accu;
-  
+
   unsigned short _rand48_mult[3] = {
     RAND48_MULT_0,
     RAND48_MULT_1,
@@ -158,11 +156,12 @@ __host__ __device__ __forceinline__ static double dev_drand48(void)
     RAND48_SEED_1,
     RAND48_SEED_2
   };
+
   return dev_erand48(_rand48_seed);
 }
 
 /** Device-callable RPowerR() function for CUDA compute kernels. */
-template <typename T>
+template < typename T >
 __host__ __device__ __forceinline__ static T RPowerR(T base, T exponent)
 {
   if (base <= 0.0)
@@ -183,9 +182,10 @@ __host__ __device__ __forceinline__ static T RPowerR(T base, T exponent)
 __host__ __device__ __forceinline__ static void AtomicMin(int *address, int val2)
 {
 #ifdef __CUDA_ARCH__
-    atomicMin(address, val2);
+  atomicMin(address, val2);
 #else
-    if(*address > val2) *address = val2;
+  if (*address > val2)
+    *address = val2;
 #endif
 }
 
@@ -201,9 +201,10 @@ __host__ __device__ __forceinline__ static void AtomicMin(int *address, int val2
 __host__ __device__ __forceinline__ static void AtomicMax(int *address, int val2)
 {
 #ifdef __CUDA_ARCH__
-    atomicMax(address, val2);
+  atomicMax(address, val2);
 #else
-    if(*address < val2) *address = val2;
+  if (*address < val2)
+    *address = val2;
 #endif
 }
 
@@ -219,16 +220,17 @@ __host__ __device__ __forceinline__ static void AtomicMax(int *address, int val2
 __host__ __device__ __forceinline__ static void AtomicMin(double *address, double val2)
 {
 #ifdef __CUDA_ARCH__
-    unsigned long long ret = __double_as_longlong(*address);
-    while(val2 < __longlong_as_double(ret))
-    {
-        unsigned long long old = ret;
-        if((ret = atomicCAS((unsigned long long *)address, old, __double_as_longlong(val2))) == old)
-            break;
-    }
-    //return __longlong_as_double(ret);
+  unsigned long long ret = __double_as_longlong(*address);
+  while (val2 < __longlong_as_double(ret))
+  {
+    unsigned long long old = ret;
+    if ((ret = atomicCAS((unsigned long long *)address, old, __double_as_longlong(val2))) == old)
+      break;
+  }
+  //return __longlong_as_double(ret);
 #else
-    if(*address > val2) *address = val2;
+  if (*address > val2)
+    *address = val2;
 #endif
 }
 
@@ -244,16 +246,17 @@ __host__ __device__ __forceinline__ static void AtomicMin(double *address, doubl
 __host__ __device__ __forceinline__ static void AtomicMax(double *address, double val2)
 {
 #ifdef __CUDA_ARCH__
-    unsigned long long ret = __double_as_longlong(*address);
-    while(val2 > __longlong_as_double(ret))
-    {
-        unsigned long long old = ret;
-        if((ret = atomicCAS((unsigned long long *)address, old, __double_as_longlong(val2))) == old)
-            break;
-    }
-    //return __longlong_as_double(ret);
+  unsigned long long ret = __double_as_longlong(*address);
+  while (val2 > __longlong_as_double(ret))
+  {
+    unsigned long long old = ret;
+    if ((ret = atomicCAS((unsigned long long *)address, old, __double_as_longlong(val2))) == old)
+      break;
+  }
+  //return __longlong_as_double(ret);
 #else
-    if(*address < val2) *address = val2;
+  if (*address < val2)
+    *address = val2;
 #endif
 }
 
@@ -266,48 +269,48 @@ __host__ __device__ __forceinline__ static void AtomicMax(double *address, doubl
  * @param array_loc original value [IN], sum result [OUT]
  * @param value value to be added [IN]
  */
-template <typename T>
+template < typename T >
 __host__ __device__ __forceinline__ static void AtomicAdd(T *array_loc, T value)
 {
-    //Define this function depending on whether it runs on GPU or CPU
+  //Define this function depending on whether it runs on GPU or CPU
 #ifdef __CUDA_ARCH__
-    atomicAdd(array_loc, value);
+  atomicAdd(array_loc, value);
 #else
-    *array_loc += value;
+  *array_loc += value;
 #endif
 }
 
- /** Helper struct for type comparison. @note Not for direct use! */
-struct SkipParallelSync {const int dummy = 0;};
+/** Helper struct for type comparison. @note Not for direct use! */
+struct SkipParallelSync { const int dummy = 0; };
 #define SKIP_PARALLEL_SYNC_cuda struct SkipParallelSync sync_struct; return sync_struct;
 
-#define PARALLEL_SYNC_cuda CUDA_ERR(cudaStreamSynchronize(0)); 
+#define PARALLEL_SYNC_cuda CUDA_ERR(cudaStreamSynchronize(0));
 
 #define PlusEquals_cuda(a, b) AtomicAdd(&(a), b)
 
- /** Helper struct for type comparison. @note Not for direct use! */
-template <typename T>
-struct ReduceMaxType {T value;};
-#define ReduceMax_cuda(a, b) struct ReduceMaxType<std::decay<decltype(a)>::type> reduce_struct {.value = b}; return reduce_struct;
+/** Helper struct for type comparison. @note Not for direct use! */
+template < typename T >
+struct ReduceMaxType { T value; };
+#define ReduceMax_cuda(a, b) struct ReduceMaxType < std::decay < decltype(a) > ::type > reduce_struct { .value = b }; return reduce_struct;
 
- /** Helper struct for type comparison. @note Not for direct use! */
-template <typename T>
-struct ReduceMinType {T value;};
-#define ReduceMin_cuda(a, b) struct ReduceMinType<std::decay<decltype(a)>::type> reduce_struct {.value = b}; return reduce_struct;
+/** Helper struct for type comparison. @note Not for direct use! */
+template < typename T >
+struct ReduceMinType { T value; };
+#define ReduceMin_cuda(a, b) struct ReduceMinType < std::decay < decltype(a) > ::type > reduce_struct { .value = b }; return reduce_struct;
 
- /** Helper struct for type comparison. @note Not for direct use! */
-template <typename T>
-struct ReduceSumType {T value;};
-#define ReduceSum_cuda(a, b) struct ReduceSumType<std::decay<decltype(a)>::type> reduce_struct {.value = b}; return reduce_struct;
+/** Helper struct for type comparison. @note Not for direct use! */
+template < typename T >
+struct ReduceSumType { T value; };
+#define ReduceSum_cuda(a, b) struct ReduceSumType < std::decay < decltype(a) > ::type > reduce_struct { .value = b }; return reduce_struct;
 
 /** A constant table for fdir (borrowed from OMP backend) */
 static const int FDIR_TABLE[6][3] = {
-  {-1, 0,  0}, // Left
-  {1,  0,  0}, // Right
-  {0, -1,  0}, // Down
-  {0,  1,  0}, // Up
-  {0,  0, -1}, // Back
-  {0,  0,  1}, // Front
+  { -1, 0, 0 }, // Left
+  { 1, 0, 0 }, // Right
+  { 0, -1, 0 }, // Down
+  { 0, 1, 0 }, // Up
+  { 0, 0, -1 }, // Back
+  { 0, 0, 1 }, // Front
 };
 
 /*--------------------------------------------------------------------------
@@ -322,20 +325,23 @@ static const int FDIR_TABLE[6][3] = {
  * @param ny The size of the second dim [IN]
  * @param nz The size of the third dim [IN]
  */
-template <typename LambdaBody>
-__global__ static void 
+template < typename LambdaBody >
+__global__ static void
 __launch_bounds__(BLOCKSIZE_MAX)
 BoxKernel(LambdaBody loop_body, const int nx, const int ny, const int nz)
 {
+  const int i = ((blockIdx.x * blockDim.x) + threadIdx.x);
 
-    const int i = ((blockIdx.x*blockDim.x)+threadIdx.x);
-    if(i >= nx)return;
-    const int j = ((blockIdx.y*blockDim.y)+threadIdx.y);
-    if(j >= ny)return;
-    const int k = ((blockIdx.z*blockDim.z)+threadIdx.z);
-    if(k >= nz)return;
-    
-    loop_body(i, j, k);
+  if (i >= nx)
+    return;
+  const int j = ((blockIdx.y * blockDim.y) + threadIdx.y);
+  if (j >= ny)
+    return;
+  const int k = ((blockIdx.z * blockDim.z) + threadIdx.z);
+  if (k >= nz)
+    return;
+
+  loop_body(i, j, k);
 }
 
 /**
@@ -348,90 +354,90 @@ BoxKernel(LambdaBody loop_body, const int nx, const int ny, const int nz)
  * @param ny The size of the second dim [IN]
  * @param nz The size of the third dim [IN]
  */
-template <typename ReduceOp, typename LambdaFun, typename T>
-__global__ static void 
+template < typename ReduceOp, typename LambdaFun, typename T >
+__global__ static void
 __launch_bounds__(BLOCKSIZE_MAX)
-DotKernel(LambdaFun loop_fun, const T init_val, T * __restrict__ rslt, 
-  const int nx, const int ny, const int nz)
+DotKernel(LambdaFun loop_fun, const T init_val, T * __restrict__ rslt,
+          const int nx, const int ny, const int nz)
 {
-    // Specialize BlockReduce for a 1D block of BLOCKSIZE_X * 1 * 1 threads on type T
+  // Specialize BlockReduce for a 1D block of BLOCKSIZE_X * 1 * 1 threads on type T
 #ifdef __CUDA_ARCH__
-    typedef cub::BlockReduce<T, BLOCKSIZE_MAX, cub::BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY, 1, 1, __CUDA_ARCH__> BlockReduce;
+  typedef cub::BlockReduce < T, BLOCKSIZE_MAX, cub::BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY, 1, 1, __CUDA_ARCH__ > BlockReduce;
 #else
-    typedef cub::BlockReduce<T, BLOCKSIZE_MAX, cub::BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY, 1, 1> BlockReduce;
+  typedef cub::BlockReduce < T, BLOCKSIZE_MAX, cub::BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY, 1, 1 > BlockReduce;
 #endif
 
-    __shared__ typename BlockReduce::TempStorage temp_storage;
+  __shared__ typename BlockReduce::TempStorage temp_storage;
 
-    const int idx = ((blockIdx.x*blockDim.x)+threadIdx.x);
+  const int idx = ((blockIdx.x * blockDim.x) + threadIdx.x);
 
-    const int i = idx % nx;
-    const int j = (idx / nx) % ny;
-    const int k = idx / (nx * ny);
-    const int ntot = nx * ny * nz;
+  const int i = idx % nx;
+  const int j = (idx / nx) % ny;
+  const int k = idx / (nx * ny);
+  const int ntot = nx * ny * nz;
 
-    T thread_data;
-    
-    // Initialize thread_data depending on reduction operation
-    if(std::is_same<ReduceOp, struct ReduceSumType<T>>::value)
-      thread_data = 0;
-    else 
-      thread_data = init_val;
+  T thread_data;
 
-    // Evaluate the loop body
-    if (idx < ntot)
-      thread_data = loop_fun(i, j, k).value;
+  // Initialize thread_data depending on reduction operation
+  if (std::is_same < ReduceOp, struct ReduceSumType < T > > ::value)
+    thread_data = 0;
+  else
+    thread_data = init_val;
 
-    // Perform reductions
-    if(std::is_same<ReduceOp, struct ReduceSumType<T>>::value)
-    {
-      // Compute the block-wide sum for thread0
-      T aggregate = BlockReduce(temp_storage).Sum(thread_data);
-
-      // Store aggregate
-      if(threadIdx.x == 0) 
-      {
-        atomicAdd(rslt, aggregate);
-      }
-    }
-    else if(std::is_same<ReduceOp, struct ReduceMaxType<T>>::value)
+  // Evaluate the loop body
+  if (idx < ntot)
+    thread_data = loop_fun(i, j, k).value;
+
+  // Perform reductions
+  if (std::is_same < ReduceOp, struct ReduceSumType < T > > ::value)
+  {
+    // Compute the block-wide sum for thread0
+    T aggregate = BlockReduce(temp_storage).Sum(thread_data);
+
+    // Store aggregate
+    if (threadIdx.x == 0)
     {
-      // Compute the block-wide sum for thread0
-      T aggregate = BlockReduce(temp_storage).Reduce(thread_data, cub::Max());
-
-      // Store aggregate
-      if(threadIdx.x == 0) 
-      {
-        AtomicMax(rslt, aggregate);
-      }
-
-      // Write to global memory directly from all threads
-      // if (idx < ntot)
-      // {
-      //   AtomicMax(rslt, thread_data);
-      // }
+      atomicAdd(rslt, aggregate);
     }
-    else if(std::is_same<ReduceOp, struct ReduceMinType<T>>::value)
+  }
+  else if (std::is_same < ReduceOp, struct ReduceMaxType < T > > ::value)
+  {
+    // Compute the block-wide sum for thread0
+    T aggregate = BlockReduce(temp_storage).Reduce(thread_data, cub::Max());
+
+    // Store aggregate
+    if (threadIdx.x == 0)
     {
-      // Compute the block-wide sum for thread0
-      T aggregate = BlockReduce(temp_storage).Reduce(thread_data, cub::Min());
-
-      // Store aggregate
-      if(threadIdx.x == 0) 
-      {
-        AtomicMin(rslt, aggregate);
-      }
-      
-      // Write to global memory directly from all threads
-      // if (idx < ntot)
-      // {
-      //   AtomicMin(rslt, thread_data);
-      // }
+      AtomicMax(rslt, aggregate);
     }
-    else 
+
+    // Write to global memory directly from all threads
+    // if (idx < ntot)
+    // {
+    //   AtomicMax(rslt, thread_data);
+    // }
+  }
+  else if (std::is_same < ReduceOp, struct ReduceMinType < T > > ::value)
+  {
+    // Compute the block-wide sum for thread0
+    T aggregate = BlockReduce(temp_storage).Reduce(thread_data, cub::Min());
+
+    // Store aggregate
+    if (threadIdx.x == 0)
     {
-      printf("ERROR at %s:%d: Invalid reduction identifier, likely a problem with a BoxLoopReduce body.", __FILE__, __LINE__);
+      AtomicMin(rslt, aggregate);
     }
+
+    // Write to global memory directly from all threads
+    // if (idx < ntot)
+    // {
+    //   AtomicMin(rslt, thread_data);
+    // }
+  }
+  else
+  {
+    printf("ERROR at %s:%d: Invalid reduction identifier, likely a problem with a BoxLoopReduce body.", __FILE__, __LINE__);
+  }
 }
 
 /*--------------------------------------------------------------------------
@@ -440,9 +446,9 @@ DotKernel(LambdaFun loop_fun, const T init_val, T * __restrict__ rslt,
 
 /**
  * @brief A macro for finding the 3D CUDA grid and block dimensions.
- * 
- * The runtime adjustment of y and z-blocksizes is based on a heuristic 
- * to improve the occupancy when launching a kernel with small grid size 
+ *
+ * The runtime adjustment of y and z-blocksizes is based on a heuristic
+ * to improve the occupancy when launching a kernel with small grid size
  * (does not have a very significant performance implications).
  *
  * @note Not for direct use!
@@ -454,27 +460,27 @@ DotKernel(LambdaFun loop_fun, const T init_val, T * __restrict__ rslt,
  * @param nz The size of the third dim [IN]
  * @param dyn_blocksize Runtime adjustment of y- and z-blocksizes [IN]
  */
-#define FindDims(grid, block, nx, ny, nz, dyn_blocksize)                            \
-{                                                                                   \
-  int blocksize_x = BLOCKSIZE_X;                                                    \
-  int blocksize_y = BLOCKSIZE_Y;                                                    \
-  int blocksize_z = BLOCKSIZE_Z;                                                    \
-  grid = dim3(((nx - 1) + blocksize_x) / blocksize_x,                               \
-      ((ny - 1) + blocksize_y) / blocksize_y,                                       \
-      ((nz - 1) + blocksize_z) / blocksize_z);                                      \
-  while(dyn_blocksize && (grid.x*grid.y*grid.z < 80)                                \
-          && ((blocksize_y * blocksize_z) >= 4 ))                                   \
-  {                                                                                 \
-      if ( blocksize_z >= 2 )                                                       \
-          blocksize_z /= 2;                                                         \
-      else                                                                          \
-          blocksize_y /= 2;                                                         \
-      grid = dim3(((nx - 1) + blocksize_x) / blocksize_x,                           \
-      ((ny - 1) + blocksize_y) / blocksize_y,                                       \
-      ((nz - 1) + blocksize_z) / blocksize_z);                                      \
-  }                                                                                 \
-  block = dim3(blocksize_x, blocksize_y, blocksize_z);                              \
-}
+#define FindDims(grid, block, nx, ny, nz, dyn_blocksize)                                              \
+        {                                                                                             \
+          int blocksize_x = BLOCKSIZE_X;                                                              \
+          int blocksize_y = BLOCKSIZE_Y;                                                              \
+          int blocksize_z = BLOCKSIZE_Z;                                                              \
+          grid = dim3(((nx - 1) + blocksize_x) / blocksize_x,                                         \
+                      ((ny - 1) + blocksize_y) / blocksize_y,                                         \
+                      ((nz - 1) + blocksize_z) / blocksize_z);                                        \
+          while (dyn_blocksize && (grid.x * grid.y * grid.z < 80)                                     \
+                 && ((blocksize_y * blocksize_z) >= 4))                                               \
+          {                                                                                           \
+            if (blocksize_z >= 2)                                                                     \
+            blocksize_z /= 2;                                                                         \
+            else                                                                                      \
+            blocksize_y /= 2;                                                                         \
+            grid = dim3(((nx - 1) + blocksize_x) / blocksize_x,                                       \
+                        ((ny - 1) + blocksize_y) / blocksize_y,                                       \
+                        ((nz - 1) + blocksize_z) / blocksize_z);                                      \
+          }                                                                                           \
+          block = dim3(blocksize_x, blocksize_y, blocksize_z);                                        \
+        }
 
 /**
  * @brief A macro for checking if cell flag data array must be reallocated.
@@ -486,765 +492,764 @@ DotKernel(LambdaFun loop_fun, const T init_val, T * __restrict__ rslt,
  * @param ny The size of the second dim [IN]
  * @param nz The size of the third dim [IN]
  */
-#define CheckCellFlagAllocation(grgeom, nx, ny, nz)                                 \
-{                                                                                   \
-  int flagdata_size = sizeof(char) * (nz * ny * nx);                                \
-  if(GrGeomSolidCellFlagDataSize(grgeom) < flagdata_size)                           \
-  {                                                                                 \
-    char *flagdata = (char*)_ctalloc_device(flagdata_size);                         \
-                                                                                    \
-    if(GrGeomSolidCellFlagDataSize(grgeom) > 0)                                     \
-      CUDA_ERR(cudaMemcpy(flagdata, GrGeomSolidCellFlagData(grgeom),                \
-        GrGeomSolidCellFlagDataSize(grgeom), cudaMemcpyDeviceToDevice));            \
-                                                                                    \
-    _tfree_device(GrGeomSolidCellFlagData(grgeom));                                 \
-    GrGeomSolidCellFlagData(grgeom) = flagdata;                                     \
-    GrGeomSolidCellFlagDataSize(grgeom) = flagdata_size;                            \
-  }                                                                                 \
-}
+#define CheckCellFlagAllocation(grgeom, nx, ny, nz)                                                         \
+        {                                                                                                   \
+          int flagdata_size = sizeof(char) * (nz * ny * nx);                                                \
+          if (GrGeomSolidCellFlagDataSize(grgeom) < flagdata_size)                                          \
+          {                                                                                                 \
+            char *flagdata = (char*)_ctalloc_device(flagdata_size);                                         \
+                                                                                                            \
+            if (GrGeomSolidCellFlagDataSize(grgeom) > 0)                                                    \
+            CUDA_ERR(cudaMemcpy(flagdata, GrGeomSolidCellFlagData(grgeom),                                  \
+                                GrGeomSolidCellFlagDataSize(grgeom), cudaMemcpyDeviceToDevice));            \
+                                                                                                            \
+            _tfree_device(GrGeomSolidCellFlagData(grgeom));                                                 \
+            GrGeomSolidCellFlagData(grgeom) = flagdata;                                                     \
+            GrGeomSolidCellFlagDataSize(grgeom) = flagdata_size;                                            \
+          }                                                                                                 \
+        }
 
- /** Loop definition for CUDA. */
-#define BoxLoopI1_cuda(i, j, k,                                                     \
-  ix, iy, iz, nx, ny, nz,                                                           \
-  i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                 \
-  loop_body)                                                                        \
-{                                                                                   \
-  if(nx > 0 && ny > 0 && nz > 0)                                                    \
-  {                                                                                 \
-    DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);     \
-                                                                                    \
-    dim3 block, grid;                                                               \
-    FindDims(grid, block, nx, ny, nz, 1);                                           \
-                                                                                    \
-    const auto &ref_i1 = i1;                                                        \
-                                                                                    \
-    auto lambda_body =                                                              \
-      GPU_LAMBDA(int i, int j, int k)                                               \
-      {                                                                             \
-        const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                     \
-          + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                              \
-                                                                                    \
-        i += ix;                                                                    \
-        j += iy;                                                                    \
-        k += iz;                                                                    \
-                                                                                    \
-        loop_body;                                                                  \
-      };                                                                            \
-                                                                                    \
-    BoxKernel<<<grid, block>>>(lambda_body, nx, ny, nz);                            \
-    CUDA_ERR(cudaPeekAtLastError());                                                \
-                                                                                    \
-    typedef function_traits<decltype(lambda_body)> traits;                          \
-    if(!std::is_same<traits::result_type, struct SkipParallelSync>::value)          \
-      CUDA_ERR(cudaStreamSynchronize(0));                                           \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
-
- /** Loop definition for CUDA. */
-#define BoxLoopI2_cuda(i, j, k,                                                     \
-  ix, iy, iz, nx, ny, nz,                                                           \
-  i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                 \
-  i2, nx2, ny2, nz2, sx2, sy2, sz2,                                                 \
-  loop_body)                                                                        \
-{                                                                                   \
-  if(nx > 0 && ny > 0 && nz > 0)                                                    \
-  {                                                                                 \
-    DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);     \
-    DeclareInc(PV_jinc_2, PV_kinc_2, nx, ny, nz, nx2, ny2, nz2, sx2, sy2, sz2);     \
-                                                                                    \
-    dim3 block, grid;                                                               \
-    FindDims(grid, block, nx, ny, nz, 1);                                           \
-                                                                                    \
-    const auto &ref_i1 = i1;                                                        \
-    const auto &ref_i2 = i2;                                                        \
-                                                                                    \
-    auto lambda_body =                                                              \
-      GPU_LAMBDA(int i, int j, int k)                                               \
-      {                                                                             \
-        const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                     \
-          + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                              \
-        const int i2 = k * PV_kinc_2 + (k * ny + j) * PV_jinc_2                     \
-          + (k * ny * nx + j * nx + i) * sx2 + ref_i2;                              \
-                                                                                    \
-        i += ix;                                                                    \
-        j += iy;                                                                    \
-        k += iz;                                                                    \
-                                                                                    \
-        loop_body;                                                                  \
-      };                                                                            \
-                                                                                    \
-    BoxKernel<<<grid, block>>>(lambda_body, nx, ny, nz);                            \
-    CUDA_ERR(cudaPeekAtLastError());                                                \
-                                                                                    \
-    typedef function_traits<decltype(lambda_body)> traits;                          \
-    if(!std::is_same<traits::result_type, struct SkipParallelSync>::value)          \
-      CUDA_ERR(cudaStreamSynchronize(0));                                           \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
+/** Loop definition for CUDA. */
+#define BoxLoopI1_cuda(i, j, k,                                                                          \
+                       ix, iy, iz, nx, ny, nz,                                                           \
+                       i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                 \
+                       loop_body)                                                                        \
+        {                                                                                                \
+          if (nx > 0 && ny > 0 && nz > 0)                                                                \
+          {                                                                                              \
+            DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);                  \
+                                                                                                         \
+            dim3 block, grid;                                                                            \
+            FindDims(grid, block, nx, ny, nz, 1);                                                        \
+                                                                                                         \
+            const auto &ref_i1 = i1;                                                                     \
+                                                                                                         \
+            auto lambda_body =                                                                           \
+              GPU_LAMBDA(int i, int j, int k)                                                            \
+            {                                                                                            \
+              const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                                    \
+                             + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                                \
+                                                                                                         \
+              i += ix;                                                                                   \
+              j += iy;                                                                                   \
+              k += iz;                                                                                   \
+                                                                                                         \
+              loop_body;                                                                                 \
+            };                                                                                           \
+                                                                                                         \
+            BoxKernel << < grid, block >> > (lambda_body, nx, ny, nz);                                   \
+            CUDA_ERR(cudaPeekAtLastError());                                                             \
+                                                                                                         \
+            typedef function_traits < decltype(lambda_body) > traits;                                    \
+            if (!std::is_same < traits::result_type, struct SkipParallelSync > ::value)                  \
+            CUDA_ERR(cudaStreamSynchronize(0));                                                          \
+          }                                                                                              \
+          (void)i; (void)j; (void)k;                                                                     \
+        }
 
- /** Loop definition for CUDA. */
-#define BoxLoopI3_cuda(i, j, k,                                                     \
-  ix, iy, iz, nx, ny, nz,                                                           \
-  i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                 \
-  i2, nx2, ny2, nz2, sx2, sy2, sz2,                                                 \
-  i3, nx3, ny3, nz3, sx3, sy3, sz3,                                                 \
-  loop_body)                                                                        \
-{                                                                                   \
-  if(nx > 0 && ny > 0 && nz > 0)                                                    \
-  {                                                                                 \
-    DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);     \
-    DeclareInc(PV_jinc_2, PV_kinc_2, nx, ny, nz, nx2, ny2, nz2, sx2, sy2, sz2);     \
-    DeclareInc(PV_jinc_3, PV_kinc_3, nx, ny, nz, nx3, ny3, nz3, sx3, sy3, sz3);     \
-                                                                                    \
-    dim3 block, grid;                                                               \
-    FindDims(grid, block, nx, ny, nz, 1);                                           \
-                                                                                    \
-    const auto &ref_i1 = i1;                                                        \
-    const auto &ref_i2 = i2;                                                        \
-    const auto &ref_i3 = i3;                                                        \
-                                                                                    \
-    auto lambda_body =                                                              \
-      GPU_LAMBDA(int i, int j, int k)                                               \
-      {                                                                             \
-        const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                     \
-          + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                              \
-        const int i2 = k * PV_kinc_2 + (k * ny + j) * PV_jinc_2                     \
-          + (k * ny * nx + j * nx + i) * sx2 + ref_i2;                              \
-        const int i3 = k * PV_kinc_3 + (k * ny + j) * PV_jinc_3                     \
-          + (k * ny * nx + j * nx + i) * sx3 + ref_i3;                              \
-                                                                                    \
-        i += ix;                                                                    \
-        j += iy;                                                                    \
-        k += iz;                                                                    \
-                                                                                    \
-        loop_body;                                                                  \
-      };                                                                            \
-                                                                                    \
-    BoxKernel<<<grid, block>>>(lambda_body, nx, ny, nz);                            \
-    CUDA_ERR(cudaPeekAtLastError());                                                \
-                                                                                    \
-    typedef function_traits<decltype(lambda_body)> traits;                          \
-    if(!std::is_same<traits::result_type, struct SkipParallelSync>::value)          \
-      CUDA_ERR(cudaStreamSynchronize(0));                                           \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
+/** Loop definition for CUDA. */
+#define BoxLoopI2_cuda(i, j, k,                                                                          \
+                       ix, iy, iz, nx, ny, nz,                                                           \
+                       i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                 \
+                       i2, nx2, ny2, nz2, sx2, sy2, sz2,                                                 \
+                       loop_body)                                                                        \
+        {                                                                                                \
+          if (nx > 0 && ny > 0 && nz > 0)                                                                \
+          {                                                                                              \
+            DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);                  \
+            DeclareInc(PV_jinc_2, PV_kinc_2, nx, ny, nz, nx2, ny2, nz2, sx2, sy2, sz2);                  \
+                                                                                                         \
+            dim3 block, grid;                                                                            \
+            FindDims(grid, block, nx, ny, nz, 1);                                                        \
+                                                                                                         \
+            const auto &ref_i1 = i1;                                                                     \
+            const auto &ref_i2 = i2;                                                                     \
+                                                                                                         \
+            auto lambda_body =                                                                           \
+              GPU_LAMBDA(int i, int j, int k)                                                            \
+            {                                                                                            \
+              const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                                    \
+                             + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                                \
+              const int i2 = k * PV_kinc_2 + (k * ny + j) * PV_jinc_2                                    \
+                             + (k * ny * nx + j * nx + i) * sx2 + ref_i2;                                \
+                                                                                                         \
+              i += ix;                                                                                   \
+              j += iy;                                                                                   \
+              k += iz;                                                                                   \
+                                                                                                         \
+              loop_body;                                                                                 \
+            };                                                                                           \
+                                                                                                         \
+            BoxKernel << < grid, block >> > (lambda_body, nx, ny, nz);                                   \
+            CUDA_ERR(cudaPeekAtLastError());                                                             \
+                                                                                                         \
+            typedef function_traits < decltype(lambda_body) > traits;                                    \
+            if (!std::is_same < traits::result_type, struct SkipParallelSync > ::value)                  \
+            CUDA_ERR(cudaStreamSynchronize(0));                                                          \
+          }                                                                                              \
+          (void)i; (void)j; (void)k;                                                                     \
+        }
 
- /** Loop definition for CUDA. */
-#define BoxLoopReduceI1_cuda(rslt, i, j, k,                                         \
-  ix, iy, iz, nx, ny, nz,                                                           \
-  i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                 \
-  loop_body)                                                                        \
-{                                                                                   \
-  if(nx > 0 && ny > 0 && nz > 0)                                                    \
-  {                                                                                 \
-    DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);     \
-                                                                                    \
-    int block = BLOCKSIZE_MAX;                                                      \
-    int grid = ((nx * ny * nz - 1) + block) / block;                                \
-                                                                                    \
-    const auto &ref_rslt = rslt;                                                    \
-    const auto &ref_i1 = i1;                                                        \
-                                                                                    \
-    auto lambda_body =                                                              \
-      GPU_LAMBDA(int i, int j, int k)                                               \
-      {                                                                             \
-        auto rslt = ref_rslt;                                                       \
-        const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                     \
-          + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                              \
-                                                                                    \
-        i += ix;                                                                    \
-        j += iy;                                                                    \
-        k += iz;                                                                    \
-                                                                                    \
-        loop_body;                                                                  \
-      };                                                                            \
-                                                                                    \
-    decltype(rslt)*ptr_rslt = (decltype(rslt)*)_talloc_device(sizeof(decltype(rslt)));\
-    MemPrefetchDeviceToHost_cuda(ptr_rslt, sizeof(decltype(rslt)), 0);              \
-    *ptr_rslt = rslt;                                                               \
-    MemPrefetchHostToDevice_cuda(ptr_rslt, sizeof(decltype(rslt)), 0);              \
-                                                                                    \
-    typedef function_traits<decltype(lambda_body)> traits;                          \
-    DotKernel<traits::result_type><<<grid, block>>>(lambda_body,                    \
-      rslt, ptr_rslt, nx, ny, nz);                                                  \
-    CUDA_ERR(cudaPeekAtLastError());                                                \
-    CUDA_ERR(cudaStreamSynchronize(0));                                             \
-                                                                                    \
-    MemPrefetchDeviceToHost_cuda(ptr_rslt, sizeof(decltype(rslt)), 0);              \
-    rslt = *ptr_rslt;                                                               \
-    _tfree_device(ptr_rslt);                                                        \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
+/** Loop definition for CUDA. */
+#define BoxLoopI3_cuda(i, j, k,                                                                          \
+                       ix, iy, iz, nx, ny, nz,                                                           \
+                       i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                 \
+                       i2, nx2, ny2, nz2, sx2, sy2, sz2,                                                 \
+                       i3, nx3, ny3, nz3, sx3, sy3, sz3,                                                 \
+                       loop_body)                                                                        \
+        {                                                                                                \
+          if (nx > 0 && ny > 0 && nz > 0)                                                                \
+          {                                                                                              \
+            DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);                  \
+            DeclareInc(PV_jinc_2, PV_kinc_2, nx, ny, nz, nx2, ny2, nz2, sx2, sy2, sz2);                  \
+            DeclareInc(PV_jinc_3, PV_kinc_3, nx, ny, nz, nx3, ny3, nz3, sx3, sy3, sz3);                  \
+                                                                                                         \
+            dim3 block, grid;                                                                            \
+            FindDims(grid, block, nx, ny, nz, 1);                                                        \
+                                                                                                         \
+            const auto &ref_i1 = i1;                                                                     \
+            const auto &ref_i2 = i2;                                                                     \
+            const auto &ref_i3 = i3;                                                                     \
+                                                                                                         \
+            auto lambda_body =                                                                           \
+              GPU_LAMBDA(int i, int j, int k)                                                            \
+            {                                                                                            \
+              const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                                    \
+                             + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                                \
+              const int i2 = k * PV_kinc_2 + (k * ny + j) * PV_jinc_2                                    \
+                             + (k * ny * nx + j * nx + i) * sx2 + ref_i2;                                \
+              const int i3 = k * PV_kinc_3 + (k * ny + j) * PV_jinc_3                                    \
+                             + (k * ny * nx + j * nx + i) * sx3 + ref_i3;                                \
+                                                                                                         \
+              i += ix;                                                                                   \
+              j += iy;                                                                                   \
+              k += iz;                                                                                   \
+                                                                                                         \
+              loop_body;                                                                                 \
+            };                                                                                           \
+                                                                                                         \
+            BoxKernel << < grid, block >> > (lambda_body, nx, ny, nz);                                   \
+            CUDA_ERR(cudaPeekAtLastError());                                                             \
+                                                                                                         \
+            typedef function_traits < decltype(lambda_body) > traits;                                    \
+            if (!std::is_same < traits::result_type, struct SkipParallelSync > ::value)                  \
+            CUDA_ERR(cudaStreamSynchronize(0));                                                          \
+          }                                                                                              \
+          (void)i; (void)j; (void)k;                                                                     \
+        }
 
- /** Loop definition for CUDA. */
-#define BoxLoopReduceI2_cuda(rslt, i, j, k,                                         \
-  ix, iy, iz, nx, ny, nz,                                                           \
-  i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                 \
-  i2, nx2, ny2, nz2, sx2, sy2, sz2,                                                 \
-  loop_body)                                                                        \
-{                                                                                   \
-  if(nx > 0 && ny > 0 && nz > 0)                                                    \
-  {                                                                                 \
-    DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);     \
-    DeclareInc(PV_jinc_2, PV_kinc_2, nx, ny, nz, nx2, ny2, nz2, sx2, sy2, sz2);     \
-                                                                                    \
-    int block = BLOCKSIZE_MAX;                                                      \
-    int grid = ((nx * ny * nz - 1) + block) / block;                                \
-                                                                                    \
-    const auto &ref_rslt = rslt;                                                    \
-    const auto &ref_i1 = i1;                                                        \
-    const auto &ref_i2 = i2;                                                        \
-                                                                                    \
-    auto lambda_body =                                                              \
-      GPU_LAMBDA(int i, int j, int k)                                               \
-      {                                                                             \
-        auto rslt = ref_rslt;                                                       \
-        const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                     \
-          + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                              \
-        const int i2 = k * PV_kinc_2 + (k * ny + j) * PV_jinc_2                     \
-          + (k * ny * nx + j * nx + i) * sx2 + ref_i2;                              \
-                                                                                    \
-        i += ix;                                                                    \
-        j += iy;                                                                    \
-        k += iz;                                                                    \
-                                                                                    \
-        loop_body;                                                                  \
-      };                                                                            \
-                                                                                    \
-    decltype(rslt)*ptr_rslt = (decltype(rslt)*)_talloc_device(sizeof(decltype(rslt)));\
-    MemPrefetchDeviceToHost_cuda(ptr_rslt, sizeof(decltype(rslt)), 0);              \
-    *ptr_rslt = rslt;                                                               \
-    MemPrefetchHostToDevice_cuda(ptr_rslt, sizeof(decltype(rslt)), 0);              \
-                                                                                    \
-    typedef function_traits<decltype(lambda_body)> traits;                          \
-    DotKernel<traits::result_type><<<grid, block>>>(lambda_body,                    \
-      rslt, ptr_rslt, nx, ny, nz);                                                  \
-    CUDA_ERR(cudaPeekAtLastError());                                                \
-    CUDA_ERR(cudaStreamSynchronize(0));                                             \
-                                                                                    \
-    MemPrefetchDeviceToHost_cuda(ptr_rslt, sizeof(decltype(rslt)), 0);              \
-    rslt = *ptr_rslt;                                                               \
-    _tfree_device(ptr_rslt);                                                        \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
+/** Loop definition for CUDA. */
+#define BoxLoopReduceI1_cuda(rslt, i, j, k,                                                                                                        \
+                             ix, iy, iz, nx, ny, nz,                                                                                               \
+                             i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                                                     \
+                             loop_body)                                                                                                            \
+        {                                                                                                                                          \
+          if (nx > 0 && ny > 0 && nz > 0)                                                                                                          \
+          {                                                                                                                                        \
+            DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);                                                            \
+                                                                                                                                                   \
+            int block = BLOCKSIZE_MAX;                                                                                                             \
+            int grid = ((nx * ny * nz - 1) + block) / block;                                                                                       \
+                                                                                                                                                   \
+            const auto &ref_rslt = rslt;                                                                                                           \
+            const auto &ref_i1 = i1;                                                                                                               \
+                                                                                                                                                   \
+            auto lambda_body =                                                                                                                     \
+              GPU_LAMBDA(int i, int j, int k)                                                                                                      \
+            {                                                                                                                                      \
+              auto rslt = ref_rslt;                                                                                                                \
+              const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                                                                              \
+                             + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                                                                          \
+                                                                                                                                                   \
+              i += ix;                                                                                                                             \
+              j += iy;                                                                                                                             \
+              k += iz;                                                                                                                             \
+                                                                                                                                                   \
+              loop_body;                                                                                                                           \
+            };                                                                                                                                     \
+                                                                                                                                                   \
+            decltype(rslt) * ptr_rslt = (decltype(rslt)*)_talloc_device(sizeof(decltype(rslt)));                                                   \
+            MemPrefetchDeviceToHost_cuda(ptr_rslt, sizeof(decltype(rslt)), 0);                                                                     \
+            *ptr_rslt = rslt;                                                                                                                      \
+            MemPrefetchHostToDevice_cuda(ptr_rslt, sizeof(decltype(rslt)), 0);                                                                     \
+                                                                                                                                                   \
+            typedef function_traits < decltype(lambda_body) > traits;                                                                              \
+            DotKernel < traits::result_type > << < grid, block >> > (lambda_body,                                                                  \
+                                                                     rslt, ptr_rslt, nx, ny, nz);                                                  \
+            CUDA_ERR(cudaPeekAtLastError());                                                                                                       \
+            CUDA_ERR(cudaStreamSynchronize(0));                                                                                                    \
+                                                                                                                                                   \
+            MemPrefetchDeviceToHost_cuda(ptr_rslt, sizeof(decltype(rslt)), 0);                                                                     \
+            rslt = *ptr_rslt;                                                                                                                      \
+            _tfree_device(ptr_rslt);                                                                                                               \
+          }                                                                                                                                        \
+          (void)i; (void)j; (void)k;                                                                                                               \
+        }
 
- /** Loop definition for CUDA. */
-#define GrGeomInLoopBoxes_cuda(i, j, k,                                             \
-  grgeom, ix, iy, iz, nx, ny, nz, loop_body)                                        \
-{                                                                                   \
-  BoxArray* boxes = GrGeomSolidInteriorBoxes(grgeom);                               \
-  int ix_bxs = BoxArrayMinCell(boxes, 0);                                           \
-  int iy_bxs = BoxArrayMinCell(boxes, 1);                                           \
-  int iz_bxs = BoxArrayMinCell(boxes, 2);                                           \
-                                                                                    \
-  int nx_bxs = BoxArrayMaxCell(boxes, 0) - ix_bxs + 1;                              \
-  int ny_bxs = BoxArrayMaxCell(boxes, 1) - iy_bxs + 1;                              \
-  int nz_bxs = BoxArrayMaxCell(boxes, 2) - iz_bxs + 1;                              \
-                                                                                    \
-  if(!(GrGeomSolidCellFlagInitialized(grgeom) & 1))                                 \
-  {                                                                                 \
-    CheckCellFlagAllocation(grgeom, nx_bxs, ny_bxs, nz_bxs);                        \
-    char *inflag = GrGeomSolidCellFlagData(grgeom);                                 \
-                                                                                    \
-    for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                    \
-    {                                                                               \
-      Box box = BoxArrayGetBox(boxes, PV_box);                                      \
-      int PV_ixl = box.lo[0];                                                       \
-      int PV_iyl = box.lo[1];                                                       \
-      int PV_izl = box.lo[2];                                                       \
-      int PV_ixu = box.up[0];                                                       \
-      int PV_iyu = box.up[1];                                                       \
-      int PV_izu = box.up[2];                                                       \
-                                                                                    \
-      if(PV_ixl <= PV_ixu && PV_iyl <= PV_iyu && PV_izl <= PV_izu)                  \
-      {                                                                             \
-        int PV_nx = PV_ixu - PV_ixl + 1;                                            \
-        int PV_ny = PV_iyu - PV_iyl + 1;                                            \
-        int PV_nz = PV_izu - PV_izl + 1;                                            \
-                                                                                    \
-        dim3 block, grid;                                                           \
-        FindDims(grid, block, PV_nx, PV_ny, PV_nz, 1);                              \
-                                                                                    \
-        Globals *globals = ::globals;                                               \
-        auto lambda_body =                                                          \
-          GPU_LAMBDA(int i, int j, int k)                                           \
-          {                                                                         \
-            i += PV_ixl;                                                            \
-            j += PV_iyl;                                                            \
-            k += PV_izl;                                                            \
-                                                                                    \
-            /* Set inflag for all cells in boxes regardless of loop limits */       \
-            inflag[(k - iz_bxs) * ny_bxs * nx_bxs +                                 \
-              (j - iy_bxs) * nx_bxs + (i - ix_bxs)] |= 1;                           \
-                                                                                    \
-            /* Only evaluate loop body if the cell is within loop limits */         \
-            if(i >= ix && j >= iy && k >= iz &&                                     \
-              i < ix + nx && j < iy + ny && k < iz + nz)                            \
-            {                                                                       \
-              loop_body;                                                            \
-            }                                                                       \
-          };                                                                        \
-                                                                                    \
-        BoxKernel<<<grid, block>>>(lambda_body, PV_nx, PV_ny, PV_nz);               \
-        CUDA_ERR(cudaPeekAtLastError());                                            \
-        CUDA_ERR(cudaStreamSynchronize(0));                                         \
-      }                                                                             \
-    }                                                                               \
-    GrGeomSolidCellFlagInitialized(grgeom) |= 1;                                    \
-  }                                                                                 \
-  else                                                                              \
-  {                                                                                 \
-    int ixl_gpu = pfmax(ix, ix_bxs);                                                \
-    int iyl_gpu = pfmax(iy, iy_bxs);                                                \
-    int izl_gpu = pfmax(iz, iz_bxs);                                                \
-    int nx_gpu = pfmin((ix + nx - 1), BoxArrayMaxCell(boxes, 0)) - ixl_gpu + 1;     \
-    int ny_gpu = pfmin((iy + ny - 1), BoxArrayMaxCell(boxes, 1)) - iyl_gpu + 1;     \
-    int nz_gpu = pfmin((iz + nz - 1), BoxArrayMaxCell(boxes, 2)) - izl_gpu + 1;     \
-                                                                                    \
-    if(nx_gpu > 0 && ny_gpu > 0 && nz_gpu > 0)                                      \
-    {                                                                               \
-      dim3 block, grid;                                                             \
-      FindDims(grid, block, nx_gpu, ny_gpu, nz_gpu, 1);                             \
-                                                                                    \
-      Globals *globals = ::globals;                                                 \
-      char *inflag = GrGeomSolidCellFlagData(grgeom);                               \
-      auto lambda_body =                                                            \
-        GPU_LAMBDA(int i, int j, int k)                                             \
-        {                                                                           \
-          i += ixl_gpu;                                                             \
-          j += iyl_gpu;                                                             \
-          k += izl_gpu;                                                             \
-          if(inflag[(k - iz_bxs) * ny_bxs * nx_bxs +                                \
-            (j - iy_bxs) * nx_bxs + (i - ix_bxs)] & 1)                              \
-          {                                                                         \
-            loop_body;                                                              \
-          }                                                                         \
-        };                                                                          \
-                                                                                    \
-      BoxKernel<<<grid, block>>>(lambda_body, nx_gpu, ny_gpu, nz_gpu);              \
-      CUDA_ERR(cudaPeekAtLastError());                                              \
-      CUDA_ERR(cudaStreamSynchronize(0));                                           \
-    }                                                                               \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
+/** Loop definition for CUDA. */
+#define BoxLoopReduceI2_cuda(rslt, i, j, k,                                                                                                        \
+                             ix, iy, iz, nx, ny, nz,                                                                                               \
+                             i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                                                     \
+                             i2, nx2, ny2, nz2, sx2, sy2, sz2,                                                                                     \
+                             loop_body)                                                                                                            \
+        {                                                                                                                                          \
+          if (nx > 0 && ny > 0 && nz > 0)                                                                                                          \
+          {                                                                                                                                        \
+            DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);                                                            \
+            DeclareInc(PV_jinc_2, PV_kinc_2, nx, ny, nz, nx2, ny2, nz2, sx2, sy2, sz2);                                                            \
+                                                                                                                                                   \
+            int block = BLOCKSIZE_MAX;                                                                                                             \
+            int grid = ((nx * ny * nz - 1) + block) / block;                                                                                       \
+                                                                                                                                                   \
+            const auto &ref_rslt = rslt;                                                                                                           \
+            const auto &ref_i1 = i1;                                                                                                               \
+            const auto &ref_i2 = i2;                                                                                                               \
+                                                                                                                                                   \
+            auto lambda_body =                                                                                                                     \
+              GPU_LAMBDA(int i, int j, int k)                                                                                                      \
+            {                                                                                                                                      \
+              auto rslt = ref_rslt;                                                                                                                \
+              const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                                                                              \
+                             + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                                                                          \
+              const int i2 = k * PV_kinc_2 + (k * ny + j) * PV_jinc_2                                                                              \
+                             + (k * ny * nx + j * nx + i) * sx2 + ref_i2;                                                                          \
+                                                                                                                                                   \
+              i += ix;                                                                                                                             \
+              j += iy;                                                                                                                             \
+              k += iz;                                                                                                                             \
+                                                                                                                                                   \
+              loop_body;                                                                                                                           \
+            };                                                                                                                                     \
+                                                                                                                                                   \
+            decltype(rslt) * ptr_rslt = (decltype(rslt)*)_talloc_device(sizeof(decltype(rslt)));                                                   \
+            MemPrefetchDeviceToHost_cuda(ptr_rslt, sizeof(decltype(rslt)), 0);                                                                     \
+            *ptr_rslt = rslt;                                                                                                                      \
+            MemPrefetchHostToDevice_cuda(ptr_rslt, sizeof(decltype(rslt)), 0);                                                                     \
+                                                                                                                                                   \
+            typedef function_traits < decltype(lambda_body) > traits;                                                                              \
+            DotKernel < traits::result_type > << < grid, block >> > (lambda_body,                                                                  \
+                                                                     rslt, ptr_rslt, nx, ny, nz);                                                  \
+            CUDA_ERR(cudaPeekAtLastError());                                                                                                       \
+            CUDA_ERR(cudaStreamSynchronize(0));                                                                                                    \
+                                                                                                                                                   \
+            MemPrefetchDeviceToHost_cuda(ptr_rslt, sizeof(decltype(rslt)), 0);                                                                     \
+            rslt = *ptr_rslt;                                                                                                                      \
+            _tfree_device(ptr_rslt);                                                                                                               \
+          }                                                                                                                                        \
+          (void)i; (void)j; (void)k;                                                                                                               \
+        }
 
- /** Loop definition for CUDA. */
-#define GrGeomSurfLoopBoxes_cuda(i, j, k, fdir, grgeom,                             \
-  ix, iy, iz, nx, ny, nz, loop_body)                                                \
-{                                                                                   \
-  for (int PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                           \
-  {                                                                                 \
-    const int *fdir = FDIR_TABLE[PV_f];                                             \
-                                                                                    \
-    BoxArray* boxes = GrGeomSolidSurfaceBoxes(grgeom, PV_f);                        \
-    int ix_bxs = BoxArrayMinCell(boxes, 0);                                         \
-    int iy_bxs = BoxArrayMinCell(boxes, 1);                                         \
-    int iz_bxs = BoxArrayMinCell(boxes, 2);                                         \
-                                                                                    \
-    int nx_bxs = BoxArrayMaxCell(boxes, 0) - ix_bxs + 1;                            \
-    int ny_bxs = BoxArrayMaxCell(boxes, 1) - iy_bxs + 1;                            \
-    int nz_bxs = BoxArrayMaxCell(boxes, 2) - iz_bxs + 1;                            \
-                                                                                    \
-    if(!(GrGeomSolidCellFlagInitialized(grgeom) & (1 << (2 + PV_f))))               \
-    {                                                                               \
-      CheckCellFlagAllocation(grgeom, nx_bxs, ny_bxs, nz_bxs);                      \
-      char *surfflag = GrGeomSolidCellFlagData(grgeom);                             \
-                                                                                    \
-      for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                  \
-      {                                                                             \
-        Box box = BoxArrayGetBox(boxes, PV_box);                                    \
-        int PV_ixl = box.lo[0];                                                     \
-        int PV_iyl = box.lo[1];                                                     \
-        int PV_izl = box.lo[2];                                                     \
-        int PV_ixu = box.up[0];                                                     \
-        int PV_iyu = box.up[1];                                                     \
-        int PV_izu = box.up[2];                                                     \
-                                                                                    \
-        if(PV_ixl <= PV_ixu && PV_iyl <= PV_iyu && PV_izl <= PV_izu)                \
-        {                                                                           \
-          int PV_nx = PV_ixu - PV_ixl + 1;                                          \
-          int PV_ny = PV_iyu - PV_iyl + 1;                                          \
-          int PV_nz = PV_izu - PV_izl + 1;                                          \
-                                                                                    \
-          dim3 block, grid;                                                         \
-          FindDims(grid, block, PV_nx, PV_ny, PV_nz, 1);                            \
-                                                                                    \
-          const int _fdir0 = fdir[0];                                               \
-          const int _fdir1 = fdir[1];                                               \
-          const int _fdir2 = fdir[2];                                               \
-                                                                                    \
-          auto lambda_body =                                                        \
-            GPU_LAMBDA(int i, int j, int k)                                         \
-            {                                                                       \
-              i += PV_ixl;                                                          \
-              j += PV_iyl;                                                          \
-              k += PV_izl;                                                          \
-                                                                                    \
-              /* Set surfflag for all cells in boxes regardless of loop limits */   \
-              surfflag[(k - iz_bxs) * ny_bxs * nx_bxs +                             \
-                (j - iy_bxs) * nx_bxs + (i - ix_bxs)] |= (1 << (2 + PV_f));         \
-                                                                                    \
-              /* Only evaluate loop body if the cell is within loop limits */       \
-              if(i >= ix && j >= iy && k >= iz &&                                   \
-                i < ix + nx && j < iy + ny && k < iz + nz)                          \
-              {                                                                     \
-                const int fdir[3] = {_fdir0, _fdir1, _fdir2};                       \
-                loop_body;                                                          \
-                (void)fdir;                                                         \
-              }                                                                     \
-            };                                                                      \
-                                                                                    \
-          BoxKernel<<<grid, block>>>(lambda_body, PV_nx, PV_ny, PV_nz);             \
-          CUDA_ERR(cudaPeekAtLastError());                                          \
-          CUDA_ERR(cudaStreamSynchronize(0));                                       \
-        }                                                                           \
-      }                                                                             \
-      GrGeomSolidCellFlagInitialized(grgeom) |= (1 << (2 + PV_f));                  \
-    }                                                                               \
-    else                                                                            \
-    {                                                                               \
-      int ixl_gpu = pfmax(ix, ix_bxs);                                              \
-      int iyl_gpu = pfmax(iy, iy_bxs);                                              \
-      int izl_gpu = pfmax(iz, iz_bxs);                                              \
-      int nx_gpu = pfmin((ix + nx - 1), BoxArrayMaxCell(boxes, 0)) - ixl_gpu + 1;   \
-      int ny_gpu = pfmin((iy + ny - 1), BoxArrayMaxCell(boxes, 1)) - iyl_gpu + 1;   \
-      int nz_gpu = pfmin((iz + nz - 1), BoxArrayMaxCell(boxes, 2)) - izl_gpu + 1;   \
-                                                                                    \
-      if(nx_gpu > 0 && ny_gpu > 0 && nz_gpu > 0)                                    \
-      {                                                                             \
-        dim3 block, grid;                                                           \
-        FindDims(grid, block, nx_gpu, ny_gpu, nz_gpu, 1);                           \
-                                                                                    \
-        char *surfflag = GrGeomSolidCellFlagData(grgeom);                           \
-                                                                                    \
-        const int _fdir0 = fdir[0];                                                 \
-        const int _fdir1 = fdir[1];                                                 \
-        const int _fdir2 = fdir[2];                                                 \
-                                                                                    \
-        auto lambda_body =                                                          \
-          GPU_LAMBDA(int i, int j, int k)                                           \
-          {                                                                         \
-            i += ixl_gpu;                                                           \
-            j += iyl_gpu;                                                           \
-            k += izl_gpu;                                                           \
-                                                                                    \
-            if(surfflag[(k - iz_bxs) * ny_bxs * nx_bxs +                            \
-              (j - iy_bxs) * nx_bxs + (i - ix_bxs)] & (1 << (2 + PV_f)))            \
-            {                                                                       \
-              const int fdir[3] = {_fdir0, _fdir1, _fdir2};                         \
-              loop_body;                                                            \
-              (void)fdir;                                                           \
-            }                                                                       \
-          };                                                                        \
-                                                                                    \
-        BoxKernel<<<grid, block>>>(lambda_body, nx_gpu, ny_gpu, nz_gpu);            \
-        CUDA_ERR(cudaPeekAtLastError());                                            \
-        CUDA_ERR(cudaStreamSynchronize(0));                                         \
-      }                                                                             \
-    }                                                                               \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
+/** Loop definition for CUDA. */
+#define GrGeomInLoopBoxes_cuda(i, j, k,                                                                          \
+                               grgeom, ix, iy, iz, nx, ny, nz, loop_body)                                        \
+        {                                                                                                        \
+          BoxArray* boxes = GrGeomSolidInteriorBoxes(grgeom);                                                    \
+          int ix_bxs = BoxArrayMinCell(boxes, 0);                                                                \
+          int iy_bxs = BoxArrayMinCell(boxes, 1);                                                                \
+          int iz_bxs = BoxArrayMinCell(boxes, 2);                                                                \
+                                                                                                                 \
+          int nx_bxs = BoxArrayMaxCell(boxes, 0) - ix_bxs + 1;                                                   \
+          int ny_bxs = BoxArrayMaxCell(boxes, 1) - iy_bxs + 1;                                                   \
+          int nz_bxs = BoxArrayMaxCell(boxes, 2) - iz_bxs + 1;                                                   \
+                                                                                                                 \
+          if (!(GrGeomSolidCellFlagInitialized(grgeom) & 1))                                                     \
+          {                                                                                                      \
+            CheckCellFlagAllocation(grgeom, nx_bxs, ny_bxs, nz_bxs);                                             \
+            char *inflag = GrGeomSolidCellFlagData(grgeom);                                                      \
+                                                                                                                 \
+            for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                                         \
+            {                                                                                                    \
+              Box box = BoxArrayGetBox(boxes, PV_box);                                                           \
+              int PV_ixl = box.lo[0];                                                                            \
+              int PV_iyl = box.lo[1];                                                                            \
+              int PV_izl = box.lo[2];                                                                            \
+              int PV_ixu = box.up[0];                                                                            \
+              int PV_iyu = box.up[1];                                                                            \
+              int PV_izu = box.up[2];                                                                            \
+                                                                                                                 \
+              if (PV_ixl <= PV_ixu && PV_iyl <= PV_iyu && PV_izl <= PV_izu)                                      \
+              {                                                                                                  \
+                int PV_nx = PV_ixu - PV_ixl + 1;                                                                 \
+                int PV_ny = PV_iyu - PV_iyl + 1;                                                                 \
+                int PV_nz = PV_izu - PV_izl + 1;                                                                 \
+                                                                                                                 \
+                dim3 block, grid;                                                                                \
+                FindDims(grid, block, PV_nx, PV_ny, PV_nz, 1);                                                   \
+                                                                                                                 \
+                Globals *globals = ::globals;                                                                    \
+                auto lambda_body =                                                                               \
+                  GPU_LAMBDA(int i, int j, int k)                                                                \
+                {                                                                                                \
+                  i += PV_ixl;                                                                                   \
+                  j += PV_iyl;                                                                                   \
+                  k += PV_izl;                                                                                   \
+                                                                                                                 \
+                  /* Set inflag for all cells in boxes regardless of loop limits */                              \
+                  inflag[(k - iz_bxs) * ny_bxs * nx_bxs +                                                        \
+                         (j - iy_bxs) * nx_bxs + (i - ix_bxs)] |= 1;                                             \
+                                                                                                                 \
+                  /* Only evaluate loop body if the cell is within loop limits */                                \
+                  if (i >= ix && j >= iy && k >= iz &&                                                           \
+                      i < ix + nx && j < iy + ny && k < iz + nz)                                                 \
+                  {                                                                                              \
+                    loop_body;                                                                                   \
+                  }                                                                                              \
+                };                                                                                               \
+                                                                                                                 \
+                BoxKernel << < grid, block >> > (lambda_body, PV_nx, PV_ny, PV_nz);                              \
+                CUDA_ERR(cudaPeekAtLastError());                                                                 \
+                CUDA_ERR(cudaStreamSynchronize(0));                                                              \
+              }                                                                                                  \
+            }                                                                                                    \
+            GrGeomSolidCellFlagInitialized(grgeom) |= 1;                                                         \
+          }                                                                                                      \
+          else                                                                                                   \
+          {                                                                                                      \
+            int ixl_gpu = pfmax(ix, ix_bxs);                                                                     \
+            int iyl_gpu = pfmax(iy, iy_bxs);                                                                     \
+            int izl_gpu = pfmax(iz, iz_bxs);                                                                     \
+            int nx_gpu = pfmin((ix + nx - 1), BoxArrayMaxCell(boxes, 0)) - ixl_gpu + 1;                          \
+            int ny_gpu = pfmin((iy + ny - 1), BoxArrayMaxCell(boxes, 1)) - iyl_gpu + 1;                          \
+            int nz_gpu = pfmin((iz + nz - 1), BoxArrayMaxCell(boxes, 2)) - izl_gpu + 1;                          \
+                                                                                                                 \
+            if (nx_gpu > 0 && ny_gpu > 0 && nz_gpu > 0)                                                          \
+            {                                                                                                    \
+              dim3 block, grid;                                                                                  \
+              FindDims(grid, block, nx_gpu, ny_gpu, nz_gpu, 1);                                                  \
+                                                                                                                 \
+              Globals *globals = ::globals;                                                                      \
+              char *inflag = GrGeomSolidCellFlagData(grgeom);                                                    \
+              auto lambda_body =                                                                                 \
+                GPU_LAMBDA(int i, int j, int k)                                                                  \
+              {                                                                                                  \
+                i += ixl_gpu;                                                                                    \
+                j += iyl_gpu;                                                                                    \
+                k += izl_gpu;                                                                                    \
+                if (inflag[(k - iz_bxs) * ny_bxs * nx_bxs +                                                      \
+                           (j - iy_bxs) * nx_bxs + (i - ix_bxs)] & 1)                                            \
+                {                                                                                                \
+                  loop_body;                                                                                     \
+                }                                                                                                \
+              };                                                                                                 \
+                                                                                                                 \
+              BoxKernel << < grid, block >> > (lambda_body, nx_gpu, ny_gpu, nz_gpu);                             \
+              CUDA_ERR(cudaPeekAtLastError());                                                                   \
+              CUDA_ERR(cudaStreamSynchronize(0));                                                                \
+            }                                                                                                    \
+          }                                                                                                      \
+          (void)i; (void)j; (void)k;                                                                             \
+        }
 
- /** Loop definition for CUDA. */
-#define GrGeomPatchLoopBoxes_cuda(i, j, k, fdir, grgeom, patch_num,                 \
-  ix, iy, iz, nx, ny, nz, loop_body)                                                \
-{                                                                                   \
-  for (int PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                           \
-  {                                                                                 \
-    const int *fdir = FDIR_TABLE[PV_f];                                             \
-                                                                                    \
-    int n_prev = 0;                                                                 \
-    BoxArray* boxes = GrGeomSolidPatchBoxes(grgeom, patch_num, PV_f);               \
-    for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                    \
-    {                                                                               \
-      Box box = BoxArrayGetBox(boxes, PV_box);                                      \
-      /* find octree and region intersection */                                     \
-      int PV_ixl = pfmax(ix, box.lo[0]);                                            \
-      int PV_iyl = pfmax(iy, box.lo[1]);                                            \
-      int PV_izl = pfmax(iz, box.lo[2]);                                            \
-      int PV_ixu = pfmin((ix + nx - 1), box.up[0]);                                 \
-      int PV_iyu = pfmin((iy + ny - 1), box.up[1]);                                 \
-      int PV_izu = pfmin((iz + nz - 1), box.up[2]);                                 \
-                                                                                    \
-      if(PV_ixl <= PV_ixu && PV_iyl <= PV_iyu && PV_izl <= PV_izu)                  \
-      {                                                                             \
-        int PV_diff_x = PV_ixu - PV_ixl;                                            \
-        int PV_diff_y = PV_iyu - PV_iyl;                                            \
-        int PV_diff_z = PV_izu - PV_izl;                                            \
-                                                                                    \
-        dim3 block, grid;                                                           \
-        FindDims(grid, block, PV_diff_x + 1, PV_diff_y + 1, PV_diff_z + 1, 1);      \
-                                                                                    \
-        int nx = PV_diff_x + 1;                                                     \
-        int ny = PV_diff_y + 1;                                                     \
-        int nz = PV_diff_z + 1;                                                     \
-                                                                                    \
-        const int _fdir0 = fdir[0];                                                 \
-        const int _fdir1 = fdir[1];                                                 \
-        const int _fdir2 = fdir[2];                                                 \
-                                                                                    \
-        auto lambda_body =                                                          \
-          GPU_LAMBDA(int i, int j, int k)                                           \
-          {                                                                         \
-            const int fdir[3] = {_fdir0, _fdir1, _fdir2};                           \
-            int ival = n_prev + k * ny * nx + j * nx + i;                           \
-                                                                                    \
-            i += PV_ixl;                                                            \
-            j += PV_iyl;                                                            \
-            k += PV_izl;                                                            \
-                                                                                    \
-            loop_body;                                                              \
-            (void)fdir;                                                             \
-          };                                                                        \
-        n_prev += nz * ny * nx;                                                     \
-                                                                                    \
-        BoxKernel<<<grid, block>>>(lambda_body, nx, ny, nz);                        \
-        CUDA_ERR(cudaPeekAtLastError());                                            \
-        CUDA_ERR(cudaStreamSynchronize(0));                                         \
-      }                                                                             \
-    }                                                                               \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
+/** Loop definition for CUDA. */
+#define GrGeomSurfLoopBoxes_cuda(i, j, k, fdir, grgeom,                                                            \
+                                 ix, iy, iz, nx, ny, nz, loop_body)                                                \
+        {                                                                                                          \
+          for (int PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                                                  \
+          {                                                                                                        \
+            const int *fdir = FDIR_TABLE[PV_f];                                                                    \
+                                                                                                                   \
+            BoxArray* boxes = GrGeomSolidSurfaceBoxes(grgeom, PV_f);                                               \
+            int ix_bxs = BoxArrayMinCell(boxes, 0);                                                                \
+            int iy_bxs = BoxArrayMinCell(boxes, 1);                                                                \
+            int iz_bxs = BoxArrayMinCell(boxes, 2);                                                                \
+                                                                                                                   \
+            int nx_bxs = BoxArrayMaxCell(boxes, 0) - ix_bxs + 1;                                                   \
+            int ny_bxs = BoxArrayMaxCell(boxes, 1) - iy_bxs + 1;                                                   \
+            int nz_bxs = BoxArrayMaxCell(boxes, 2) - iz_bxs + 1;                                                   \
+                                                                                                                   \
+            if (!(GrGeomSolidCellFlagInitialized(grgeom) & (1 << (2 + PV_f))))                                     \
+            {                                                                                                      \
+              CheckCellFlagAllocation(grgeom, nx_bxs, ny_bxs, nz_bxs);                                             \
+              char *surfflag = GrGeomSolidCellFlagData(grgeom);                                                    \
+                                                                                                                   \
+              for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                                         \
+              {                                                                                                    \
+                Box box = BoxArrayGetBox(boxes, PV_box);                                                           \
+                int PV_ixl = box.lo[0];                                                                            \
+                int PV_iyl = box.lo[1];                                                                            \
+                int PV_izl = box.lo[2];                                                                            \
+                int PV_ixu = box.up[0];                                                                            \
+                int PV_iyu = box.up[1];                                                                            \
+                int PV_izu = box.up[2];                                                                            \
+                                                                                                                   \
+                if (PV_ixl <= PV_ixu && PV_iyl <= PV_iyu && PV_izl <= PV_izu)                                      \
+                {                                                                                                  \
+                  int PV_nx = PV_ixu - PV_ixl + 1;                                                                 \
+                  int PV_ny = PV_iyu - PV_iyl + 1;                                                                 \
+                  int PV_nz = PV_izu - PV_izl + 1;                                                                 \
+                                                                                                                   \
+                  dim3 block, grid;                                                                                \
+                  FindDims(grid, block, PV_nx, PV_ny, PV_nz, 1);                                                   \
+                                                                                                                   \
+                  const int _fdir0 = fdir[0];                                                                      \
+                  const int _fdir1 = fdir[1];                                                                      \
+                  const int _fdir2 = fdir[2];                                                                      \
+                                                                                                                   \
+                  auto lambda_body =                                                                               \
+                    GPU_LAMBDA(int i, int j, int k)                                                                \
+                  {                                                                                                \
+                    i += PV_ixl;                                                                                   \
+                    j += PV_iyl;                                                                                   \
+                    k += PV_izl;                                                                                   \
+                                                                                                                   \
+                    /* Set surfflag for all cells in boxes regardless of loop limits */                            \
+                    surfflag[(k - iz_bxs) * ny_bxs * nx_bxs +                                                      \
+                             (j - iy_bxs) * nx_bxs + (i - ix_bxs)] |= (1 << (2 + PV_f));                           \
+                                                                                                                   \
+                    /* Only evaluate loop body if the cell is within loop limits */                                \
+                    if (i >= ix && j >= iy && k >= iz &&                                                           \
+                        i < ix + nx && j < iy + ny && k < iz + nz)                                                 \
+                    {                                                                                              \
+                      const int fdir[3] = { _fdir0, _fdir1, _fdir2 };                                              \
+                      loop_body;                                                                                   \
+                      (void)fdir;                                                                                  \
+                    }                                                                                              \
+                  };                                                                                               \
+                                                                                                                   \
+                  BoxKernel << < grid, block >> > (lambda_body, PV_nx, PV_ny, PV_nz);                              \
+                  CUDA_ERR(cudaPeekAtLastError());                                                                 \
+                  CUDA_ERR(cudaStreamSynchronize(0));                                                              \
+                }                                                                                                  \
+              }                                                                                                    \
+              GrGeomSolidCellFlagInitialized(grgeom) |= (1 << (2 + PV_f));                                         \
+            }                                                                                                      \
+            else                                                                                                   \
+            {                                                                                                      \
+              int ixl_gpu = pfmax(ix, ix_bxs);                                                                     \
+              int iyl_gpu = pfmax(iy, iy_bxs);                                                                     \
+              int izl_gpu = pfmax(iz, iz_bxs);                                                                     \
+              int nx_gpu = pfmin((ix + nx - 1), BoxArrayMaxCell(boxes, 0)) - ixl_gpu + 1;                          \
+              int ny_gpu = pfmin((iy + ny - 1), BoxArrayMaxCell(boxes, 1)) - iyl_gpu + 1;                          \
+              int nz_gpu = pfmin((iz + nz - 1), BoxArrayMaxCell(boxes, 2)) - izl_gpu + 1;                          \
+                                                                                                                   \
+              if (nx_gpu > 0 && ny_gpu > 0 && nz_gpu > 0)                                                          \
+              {                                                                                                    \
+                dim3 block, grid;                                                                                  \
+                FindDims(grid, block, nx_gpu, ny_gpu, nz_gpu, 1);                                                  \
+                                                                                                                   \
+                char *surfflag = GrGeomSolidCellFlagData(grgeom);                                                  \
+                                                                                                                   \
+                const int _fdir0 = fdir[0];                                                                        \
+                const int _fdir1 = fdir[1];                                                                        \
+                const int _fdir2 = fdir[2];                                                                        \
+                                                                                                                   \
+                auto lambda_body =                                                                                 \
+                  GPU_LAMBDA(int i, int j, int k)                                                                  \
+                {                                                                                                  \
+                  i += ixl_gpu;                                                                                    \
+                  j += iyl_gpu;                                                                                    \
+                  k += izl_gpu;                                                                                    \
+                                                                                                                   \
+                  if (surfflag[(k - iz_bxs) * ny_bxs * nx_bxs +                                                    \
+                               (j - iy_bxs) * nx_bxs + (i - ix_bxs)] & (1 << (2 + PV_f)))                          \
+                  {                                                                                                \
+                    const int fdir[3] = { _fdir0, _fdir1, _fdir2 };                                                \
+                    loop_body;                                                                                     \
+                    (void)fdir;                                                                                    \
+                  }                                                                                                \
+                };                                                                                                 \
+                                                                                                                   \
+                BoxKernel << < grid, block >> > (lambda_body, nx_gpu, ny_gpu, nz_gpu);                             \
+                CUDA_ERR(cudaPeekAtLastError());                                                                   \
+                CUDA_ERR(cudaStreamSynchronize(0));                                                                \
+              }                                                                                                    \
+            }                                                                                                      \
+          }                                                                                                        \
+          (void)i; (void)j; (void)k;                                                                               \
+        }
 
- /** Loop definition for CUDA. */
-#define GrGeomPatchLoopBoxesNoFdir_cuda(i, j, k, grgeom, patch_num, ovrlnd,         \
-  ix, iy, iz, nx, ny, nz, locals, setup,                                            \
-  f_left, f_right, f_down, f_up, f_back, f_front, finalize)                         \
-{                                                                                   \
-  int n_ival = 0;                                                                   \
-  for (int PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                           \
-  {                                                                                 \
-    BoxArray* boxes = GrGeomSolidPatchBoxes(grgeom, patch_num, PV_f);               \
-                                                                                    \
-    int ix_bxs = BoxArrayMinCell(boxes, 0);                                         \
-    int iy_bxs = BoxArrayMinCell(boxes, 1);                                         \
-    int iz_bxs = BoxArrayMinCell(boxes, 2);                                         \
-                                                                                    \
-    int nx_bxs = BoxArrayMaxCell(boxes, 0) - ix_bxs + 1;                            \
-    int ny_bxs = BoxArrayMaxCell(boxes, 1) - iy_bxs + 1;                            \
-    int nz_bxs = BoxArrayMaxCell(boxes, 2) - iz_bxs + 1;                            \
-                                                                                    \
-    int patch_loc;                                                                  \
-    if(ovrlnd)                                                                      \
-      patch_loc = GrGeomSolidNumPatches(grgeom) + patch_num;                        \
-    else                                                                            \
-      patch_loc = patch_num;                                                        \
-                                                                                    \
-    int *ptr_ival = GrGeomSolidCellIval(grgeom, patch_loc, PV_f);                   \
-    if(!(ptr_ival))                                                                 \
-    {                                                                               \
-      GrGeomSolidCellIval(grgeom, patch_loc, PV_f) =                                \
-        (int*)_talloc_device(sizeof(int) * nx_bxs * ny_bxs * nz_bxs);               \
-                                                                                    \
-      ptr_ival = GrGeomSolidCellIval(grgeom, patch_loc, PV_f);                      \
-      for (int idx = 0; idx < nx_bxs * ny_bxs * nz_bxs; idx++)                      \
-        ptr_ival[idx] = -1;                                                         \
-                                                                                    \
-      for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                  \
-      {                                                                             \
-        Box box = BoxArrayGetBox(boxes, PV_box);                                    \
-        int PV_ixl = pfmax(ix, box.lo[0]);                                          \
-        int PV_iyl = pfmax(iy, box.lo[1]);                                          \
-        int PV_izl = pfmax(iz, box.lo[2]);                                          \
-        int PV_ixu = pfmin((ix + nx - 1), box.up[0]);                               \
-        int PV_iyu = pfmin((iy + ny - 1), box.up[1]);                               \
-        int PV_izu = pfmin((iz + nz - 1), box.up[2]);                               \
-                                                                                    \
-        for (k = PV_izl; k <= PV_izu; k++)                                          \
-          for (j = PV_iyl; j <= PV_iyu; j++)                                        \
-            for (i = PV_ixl; i <= PV_ixu; i++)                                      \
-            {                                                                       \
-              UNPACK(locals);                                                       \
-              setup;                                                                \
-              switch(PV_f)                                                          \
-              {                                                                     \
-                f_left;                                                             \
-                f_right;                                                            \
-                f_down;                                                             \
-                f_up;                                                               \
-                f_back;                                                             \
-                f_front;                                                            \
-              }                                                                     \
-              finalize;                                                             \
-              ptr_ival[(k - iz_bxs) * ny_bxs * nx_bxs + (j - iy_bxs) *              \
-                nx_bxs + (i - ix_bxs)] = n_ival++;                                  \
-            }                                                                       \
-      }                                                                             \
-    }                                                                               \
-    else                                                                            \
-    {                                                                               \
-      int ixl_gpu = pfmax(ix, ix_bxs);                                              \
-      int iyl_gpu = pfmax(iy, iy_bxs);                                              \
-      int izl_gpu = pfmax(iz, iz_bxs);                                              \
-      int nx_gpu = pfmin((ix + nx - 1), BoxArrayMaxCell(boxes, 0)) - ixl_gpu + 1;   \
-      int ny_gpu = pfmin((iy + ny - 1), BoxArrayMaxCell(boxes, 1)) - iyl_gpu + 1;   \
-      int nz_gpu = pfmin((iz + nz - 1), BoxArrayMaxCell(boxes, 2)) - izl_gpu + 1;   \
-                                                                                    \
-      if(nx_gpu > 0 && ny_gpu > 0 && nz_gpu > 0)                                    \
-      {                                                                             \
-        dim3 block, grid;                                                           \
-        FindDims(grid, block, nx_gpu, ny_gpu, nz_gpu, 1);                           \
-                                                                                    \
-        auto lambda_body =                                                          \
-          GPU_LAMBDA(int i, int j, int k)                                           \
-          {                                                                         \
-            i += ixl_gpu;                                                           \
-            j += iyl_gpu;                                                           \
-            k += izl_gpu;                                                           \
-                                                                                    \
-            int ival = ptr_ival[(k - iz_bxs) * ny_bxs * nx_bxs +                    \
-              (j - iy_bxs) * nx_bxs + (i - ix_bxs)];                                \
-            if(ival >= 0)                                                           \
-            {                                                                       \
-              UNPACK(locals);                                                       \
-              setup;                                                                \
-              switch(PV_f)                                                          \
-              {                                                                     \
-                f_left;                                                             \
-                f_right;                                                            \
-                f_down;                                                             \
-                f_up;                                                               \
-                f_back;                                                             \
-                f_front;                                                            \
-              }                                                                     \
-              finalize;                                                             \
-            }                                                                       \
-          };                                                                        \
-                                                                                    \
-        BoxKernel<<<grid, block>>>(lambda_body, nx_gpu, ny_gpu, nz_gpu);            \
-        CUDA_ERR(cudaPeekAtLastError());                                            \
-        CUDA_ERR(cudaStreamSynchronize(0));                                         \
-      }                                                                             \
-    }                                                                               \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
+/** Loop definition for CUDA. */
+#define GrGeomPatchLoopBoxes_cuda(i, j, k, fdir, grgeom, patch_num,                                                 \
+                                  ix, iy, iz, nx, ny, nz, loop_body)                                                \
+        {                                                                                                           \
+          for (int PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                                                   \
+          {                                                                                                         \
+            const int *fdir = FDIR_TABLE[PV_f];                                                                     \
+                                                                                                                    \
+            int n_prev = 0;                                                                                         \
+            BoxArray* boxes = GrGeomSolidPatchBoxes(grgeom, patch_num, PV_f);                                       \
+            for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                                            \
+            {                                                                                                       \
+              Box box = BoxArrayGetBox(boxes, PV_box);                                                              \
+              /* find octree and region intersection */                                                             \
+              int PV_ixl = pfmax(ix, box.lo[0]);                                                                    \
+              int PV_iyl = pfmax(iy, box.lo[1]);                                                                    \
+              int PV_izl = pfmax(iz, box.lo[2]);                                                                    \
+              int PV_ixu = pfmin((ix + nx - 1), box.up[0]);                                                         \
+              int PV_iyu = pfmin((iy + ny - 1), box.up[1]);                                                         \
+              int PV_izu = pfmin((iz + nz - 1), box.up[2]);                                                         \
+                                                                                                                    \
+              if (PV_ixl <= PV_ixu && PV_iyl <= PV_iyu && PV_izl <= PV_izu)                                         \
+              {                                                                                                     \
+                int PV_diff_x = PV_ixu - PV_ixl;                                                                    \
+                int PV_diff_y = PV_iyu - PV_iyl;                                                                    \
+                int PV_diff_z = PV_izu - PV_izl;                                                                    \
+                                                                                                                    \
+                dim3 block, grid;                                                                                   \
+                FindDims(grid, block, PV_diff_x + 1, PV_diff_y + 1, PV_diff_z + 1, 1);                              \
+                                                                                                                    \
+                int nx = PV_diff_x + 1;                                                                             \
+                int ny = PV_diff_y + 1;                                                                             \
+                int nz = PV_diff_z + 1;                                                                             \
+                                                                                                                    \
+                const int _fdir0 = fdir[0];                                                                         \
+                const int _fdir1 = fdir[1];                                                                         \
+                const int _fdir2 = fdir[2];                                                                         \
+                                                                                                                    \
+                auto lambda_body =                                                                                  \
+                  GPU_LAMBDA(int i, int j, int k)                                                                   \
+                {                                                                                                   \
+                  const int fdir[3] = { _fdir0, _fdir1, _fdir2 };                                                   \
+                  int ival = n_prev + k * ny * nx + j * nx + i;                                                     \
+                                                                                                                    \
+                  i += PV_ixl;                                                                                      \
+                  j += PV_iyl;                                                                                      \
+                  k += PV_izl;                                                                                      \
+                                                                                                                    \
+                  loop_body;                                                                                        \
+                  (void)fdir;                                                                                       \
+                };                                                                                                  \
+                n_prev += nz * ny * nx;                                                                             \
+                                                                                                                    \
+                BoxKernel << < grid, block >> > (lambda_body, nx, ny, nz);                                          \
+                CUDA_ERR(cudaPeekAtLastError());                                                                    \
+                CUDA_ERR(cudaStreamSynchronize(0));                                                                 \
+              }                                                                                                     \
+            }                                                                                                       \
+          }                                                                                                         \
+          (void)i; (void)j; (void)k;                                                                                \
+        }
 
- /** Loop definition for CUDA. */
-#define GrGeomOctreeExteriorNodeLoop_cuda(i, j, k, node, octree, level,             \
-  ix, iy, iz, nx, ny, nz, val_test, loop_body)                                      \
-{                                                                                   \
-  int PV_i, PV_j, PV_k, PV_l;                                                       \
-  int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                               \
-                                                                                    \
-  PV_i = i;                                                                         \
-  PV_j = j;                                                                         \
-  PV_k = k;                                                                         \
-                                                                                    \
-  GrGeomOctreeExteriorLoop(PV_i, PV_j, PV_k, PV_l, node, octree, level, val_test,   \
-  {                                                                                 \
-    if ((PV_i >= ix) && (PV_i < (ix + nx)) &&                                       \
-        (PV_j >= iy) && (PV_j < (iy + ny)) &&                                       \
-        (PV_k >= iz) && (PV_k < (iz + nz)))                                         \
-    {                                                                               \
-      i = PV_i;                                                                     \
-      j = PV_j;                                                                     \
-      k = PV_k;                                                                     \
-      loop_body;                                                                    \
-    }                                                                               \
-  },                                                                                \
-  {                                                                                 \
-    /* find octree and region intersection */                                       \
-    PV_ixl = pfmax(ix, PV_i);                                                       \
-    PV_iyl = pfmax(iy, PV_j);                                                       \
-    PV_izl = pfmax(iz, PV_k);                                                       \
-    PV_ixu = pfmin((ix + nx), (PV_i + (int)PV_inc));                                \
-    PV_iyu = pfmin((iy + ny), (PV_j + (int)PV_inc));                                \
-    PV_izu = pfmin((iz + nz), (PV_k + (int)PV_inc));                                \
-                                                                                    \
-    if(PV_ixl < PV_ixu && PV_iyl < PV_iyu && PV_izl < PV_izu)                       \
-    {                                                                               \
-      const int PV_diff_x = PV_ixu - PV_ixl;                                        \
-      const int PV_diff_y = PV_iyu - PV_iyl;                                        \
-      const int PV_diff_z = PV_izu - PV_izl;                                        \
-                                                                                    \
-      dim3 block;                                                                   \
-      dim3 grid;                                                                    \
-      FindDims(grid, block, PV_diff_x, PV_diff_y, PV_diff_z, 1);                    \
-                                                                                    \
-      auto lambda_body =                                                            \
-        GPU_LAMBDA(int i, int j, int k)                                             \
-        {                                                                           \
-          i += PV_ixl;                                                              \
-          j += PV_iyl;                                                              \
-          k += PV_izl;                                                              \
-                                                                                    \
-          loop_body;                                                                \
-        };                                                                          \
-                                                                                    \
-      (BoxKernel<<<grid, block>>>(lambda_body, PV_diff_x, PV_diff_y, PV_diff_z));   \
-      CUDA_ERR(cudaPeekAtLastError());                                              \
-      CUDA_ERR(cudaStreamSynchronize(0));                                           \
-    }                                                                               \
-    i = PV_ixu;                                                                     \
-    j = PV_iyu;                                                                     \
-    k = PV_izu;                                                                     \
-  })                                                                                \
-  (void)i;(void)j;(void)k;                                                          \
-}
+/** Loop definition for CUDA. */
+#define GrGeomPatchLoopBoxesNoFdir_cuda(i, j, k, grgeom, patch_num, ovrlnd,                                               \
+                                        ix, iy, iz, nx, ny, nz, locals, setup,                                            \
+                                        f_left, f_right, f_down, f_up, f_back, f_front, finalize)                         \
+        {                                                                                                                 \
+          int n_ival = 0;                                                                                                 \
+          for (int PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                                                         \
+          {                                                                                                               \
+            BoxArray* boxes = GrGeomSolidPatchBoxes(grgeom, patch_num, PV_f);                                             \
+                                                                                                                          \
+            int ix_bxs = BoxArrayMinCell(boxes, 0);                                                                       \
+            int iy_bxs = BoxArrayMinCell(boxes, 1);                                                                       \
+            int iz_bxs = BoxArrayMinCell(boxes, 2);                                                                       \
+                                                                                                                          \
+            int nx_bxs = BoxArrayMaxCell(boxes, 0) - ix_bxs + 1;                                                          \
+            int ny_bxs = BoxArrayMaxCell(boxes, 1) - iy_bxs + 1;                                                          \
+            int nz_bxs = BoxArrayMaxCell(boxes, 2) - iz_bxs + 1;                                                          \
+                                                                                                                          \
+            int patch_loc;                                                                                                \
+            if (ovrlnd)                                                                                                   \
+            patch_loc = GrGeomSolidNumPatches(grgeom) +patch_num;                                                         \
+            else                                                                                                          \
+            patch_loc = patch_num;                                                                                        \
+                                                                                                                          \
+            int *ptr_ival = GrGeomSolidCellIval(grgeom, patch_loc, PV_f);                                                 \
+            if (!(ptr_ival))                                                                                              \
+            {                                                                                                             \
+              GrGeomSolidCellIval(grgeom, patch_loc, PV_f) =                                                              \
+                (int*)_talloc_device(sizeof(int) * nx_bxs * ny_bxs * nz_bxs);                                             \
+                                                                                                                          \
+              ptr_ival = GrGeomSolidCellIval(grgeom, patch_loc, PV_f);                                                    \
+              for (int idx = 0; idx < nx_bxs * ny_bxs * nz_bxs; idx++)                                                    \
+              ptr_ival[idx] = -1;                                                                                         \
+                                                                                                                          \
+              for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                                                \
+              {                                                                                                           \
+                Box box = BoxArrayGetBox(boxes, PV_box);                                                                  \
+                int PV_ixl = pfmax(ix, box.lo[0]);                                                                        \
+                int PV_iyl = pfmax(iy, box.lo[1]);                                                                        \
+                int PV_izl = pfmax(iz, box.lo[2]);                                                                        \
+                int PV_ixu = pfmin((ix + nx - 1), box.up[0]);                                                             \
+                int PV_iyu = pfmin((iy + ny - 1), box.up[1]);                                                             \
+                int PV_izu = pfmin((iz + nz - 1), box.up[2]);                                                             \
+                                                                                                                          \
+                for (k = PV_izl; k <= PV_izu; k++)                                                                        \
+                for (j = PV_iyl; j <= PV_iyu; j++)                                                                        \
+                for (i = PV_ixl; i <= PV_ixu; i++)                                                                        \
+                {                                                                                                         \
+                  UNPACK(locals);                                                                                         \
+                  setup;                                                                                                  \
+                  switch (PV_f)                                                                                           \
+                  {                                                                                                       \
+                  f_left;                                                                                                 \
+                  f_right;                                                                                                \
+                  f_down;                                                                                                 \
+                  f_up;                                                                                                   \
+                  f_back;                                                                                                 \
+                  f_front;                                                                                                \
+                  }                                                                                                       \
+                  finalize;                                                                                               \
+                  ptr_ival[(k - iz_bxs) * ny_bxs * nx_bxs + (j - iy_bxs) *                                                \
+                           nx_bxs + (i - ix_bxs)] = n_ival++;                                                             \
+                }                                                                                                         \
+              }                                                                                                           \
+            }                                                                                                             \
+            else                                                                                                          \
+            {                                                                                                             \
+              int ixl_gpu = pfmax(ix, ix_bxs);                                                                            \
+              int iyl_gpu = pfmax(iy, iy_bxs);                                                                            \
+              int izl_gpu = pfmax(iz, iz_bxs);                                                                            \
+              int nx_gpu = pfmin((ix + nx - 1), BoxArrayMaxCell(boxes, 0)) - ixl_gpu + 1;                                 \
+              int ny_gpu = pfmin((iy + ny - 1), BoxArrayMaxCell(boxes, 1)) - iyl_gpu + 1;                                 \
+              int nz_gpu = pfmin((iz + nz - 1), BoxArrayMaxCell(boxes, 2)) - izl_gpu + 1;                                 \
+                                                                                                                          \
+              if (nx_gpu > 0 && ny_gpu > 0 && nz_gpu > 0)                                                                 \
+              {                                                                                                           \
+                dim3 block, grid;                                                                                         \
+                FindDims(grid, block, nx_gpu, ny_gpu, nz_gpu, 1);                                                         \
+                                                                                                                          \
+                auto lambda_body =                                                                                        \
+                  GPU_LAMBDA(int i, int j, int k)                                                                         \
+                {                                                                                                         \
+                  i += ixl_gpu;                                                                                           \
+                  j += iyl_gpu;                                                                                           \
+                  k += izl_gpu;                                                                                           \
+                                                                                                                          \
+                  int ival = ptr_ival[(k - iz_bxs) * ny_bxs * nx_bxs +                                                    \
+                                      (j - iy_bxs) * nx_bxs + (i - ix_bxs)];                                              \
+                  if (ival >= 0)                                                                                          \
+                  {                                                                                                       \
+                    UNPACK(locals);                                                                                       \
+                    setup;                                                                                                \
+                    switch (PV_f)                                                                                         \
+                    {                                                                                                     \
+                    f_left;                                                                                               \
+                    f_right;                                                                                              \
+                    f_down;                                                                                               \
+                    f_up;                                                                                                 \
+                    f_back;                                                                                               \
+                    f_front;                                                                                              \
+                    }                                                                                                     \
+                    finalize;                                                                                             \
+                  }                                                                                                       \
+                };                                                                                                        \
+                                                                                                                          \
+                BoxKernel << < grid, block >> > (lambda_body, nx_gpu, ny_gpu, nz_gpu);                                    \
+                CUDA_ERR(cudaPeekAtLastError());                                                                          \
+                CUDA_ERR(cudaStreamSynchronize(0));                                                                       \
+              }                                                                                                           \
+            }                                                                                                             \
+          }                                                                                                               \
+          (void)i; (void)j; (void)k;                                                                                      \
+        }
 
- /** Loop definition for CUDA. */
-#define GrGeomOutLoop_cuda(i, j, k, grgeom, r,                                      \
-  ix, iy, iz, nx, ny, nz, body)                                                     \
-{                                                                                   \
-  if(nx > 0 && ny > 0 && nz > 0)                                                    \
-  {                                                                                 \
-    if(!(GrGeomSolidCellFlagInitialized(grgeom) & (1 << 1)))                        \
-    {                                                                               \
-      CheckCellFlagAllocation(grgeom, nx, ny, nz);                                  \
-      char *outflag = GrGeomSolidCellFlagData(grgeom);                              \
-                                                                                    \
-      GrGeomOctree  *PV_node;                                                       \
-      double PV_ref = pow(2.0, r);                                                  \
-                                                                                    \
-      i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;                                \
-      j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;                                \
-      k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;                                \
-      GrGeomOctreeExteriorNodeLoop(i, j, k, PV_node,                                \
-                                   GrGeomSolidData(grgeom),                         \
-                                   GrGeomSolidOctreeBGLevel(grgeom) + r,            \
-                                   ix, iy, iz, nx, ny, nz,                          \
-                                   TRUE,                                            \
-      {                                                                             \
-        body;                                                                       \
-        outflag[(k - iz) * ny * nx + (j - iy) * nx + (i - ix)] |= (1 << 1);         \
-      });                                                                           \
-      GrGeomSolidCellFlagInitialized(grgeom) |= (1 << 1);                           \
-    }                                                                               \
-    else                                                                            \
-    {                                                                               \
-      dim3 block, grid;                                                             \
-      FindDims(grid, block, nx, ny, nz, 1);                                         \
-                                                                                    \
-      char *outflag = GrGeomSolidCellFlagData(grgeom);                              \
-      auto lambda_body =                                                            \
-        GPU_LAMBDA(int i, int j, int k)                                             \
-        {                                                                           \
-          i += ix;                                                                  \
-          j += iy;                                                                  \
-          k += iz;                                                                  \
-                                                                                    \
-          if(outflag[(k - iz) * ny * nx + (j - iy) * nx + (i - ix)] & (1 << 1))     \
-          {                                                                         \
-            body;                                                                   \
-          }                                                                         \
-        };                                                                          \
-                                                                                    \
-      BoxKernel<<<grid, block>>>(lambda_body, nx, ny, nz);                          \
-      CUDA_ERR(cudaPeekAtLastError());                                              \
-      CUDA_ERR(cudaStreamSynchronize(0));                                           \
-    }                                                                               \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
+/** Loop definition for CUDA. */
+#define GrGeomOctreeExteriorNodeLoop_cuda(i, j, k, node, octree, level,                                                     \
+                                          ix, iy, iz, nx, ny, nz, val_test, loop_body)                                      \
+        {                                                                                                                   \
+          int PV_i, PV_j, PV_k, PV_l;                                                                                       \
+          int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                                                               \
+                                                                                                                            \
+          PV_i = i;                                                                                                         \
+          PV_j = j;                                                                                                         \
+          PV_k = k;                                                                                                         \
+                                                                                                                            \
+          GrGeomOctreeExteriorLoop(PV_i, PV_j, PV_k, PV_l, node, octree, level, val_test,                                   \
+    {                                                                                                                       \
+      if ((PV_i >= ix) && (PV_i < (ix + nx)) &&                                                                             \
+          (PV_j >= iy) && (PV_j < (iy + ny)) &&                                                                             \
+          (PV_k >= iz) && (PV_k < (iz + nz)))                                                                               \
+      {                                                                                                                     \
+        i = PV_i;                                                                                                           \
+        j = PV_j;                                                                                                           \
+        k = PV_k;                                                                                                           \
+        loop_body;                                                                                                          \
+      }                                                                                                                     \
+    },                                                                                                                      \
+    {                                                                                                                       \
+      /* find octree and region intersection */                                                                             \
+      PV_ixl = pfmax(ix, PV_i);                                                                                             \
+      PV_iyl = pfmax(iy, PV_j);                                                                                             \
+      PV_izl = pfmax(iz, PV_k);                                                                                             \
+      PV_ixu = pfmin((ix + nx), (PV_i + (int)PV_inc));                                                                      \
+      PV_iyu = pfmin((iy + ny), (PV_j + (int)PV_inc));                                                                      \
+      PV_izu = pfmin((iz + nz), (PV_k + (int)PV_inc));                                                                      \
+                                                                                                                            \
+      if (PV_ixl < PV_ixu && PV_iyl < PV_iyu && PV_izl < PV_izu)                                                            \
+      {                                                                                                                     \
+        const int PV_diff_x = PV_ixu - PV_ixl;                                                                              \
+        const int PV_diff_y = PV_iyu - PV_iyl;                                                                              \
+        const int PV_diff_z = PV_izu - PV_izl;                                                                              \
+                                                                                                                            \
+        dim3 block;                                                                                                         \
+        dim3 grid;                                                                                                          \
+        FindDims(grid, block, PV_diff_x, PV_diff_y, PV_diff_z, 1);                                                          \
+                                                                                                                            \
+        auto lambda_body =                                                                                                  \
+          GPU_LAMBDA(int i, int j, int k)                                                                                   \
+        {                                                                                                                   \
+          i += PV_ixl;                                                                                                      \
+          j += PV_iyl;                                                                                                      \
+          k += PV_izl;                                                                                                      \
+                                                                                                                            \
+          loop_body;                                                                                                        \
+        };                                                                                                                  \
+                                                                                                                            \
+        (BoxKernel << < grid, block >> > (lambda_body, PV_diff_x, PV_diff_y, PV_diff_z));                                   \
+        CUDA_ERR(cudaPeekAtLastError());                                                                                    \
+        CUDA_ERR(cudaStreamSynchronize(0));                                                                                 \
+      }                                                                                                                     \
+      i = PV_ixu;                                                                                                           \
+      j = PV_iyu;                                                                                                           \
+      k = PV_izu;                                                                                                           \
+    })                                                                                                                      \
+          (void)i; (void)j; (void)k;                                                                                        \
+        }
 
+/** Loop definition for CUDA. */
+#define GrGeomOutLoop_cuda(i, j, k, grgeom, r,                                                               \
+                           ix, iy, iz, nx, ny, nz, body)                                                     \
+        {                                                                                                    \
+          if (nx > 0 && ny > 0 && nz > 0)                                                                    \
+          {                                                                                                  \
+            if (!(GrGeomSolidCellFlagInitialized(grgeom) & (1 << 1)))                                        \
+            {                                                                                                \
+              CheckCellFlagAllocation(grgeom, nx, ny, nz);                                                   \
+              char *outflag = GrGeomSolidCellFlagData(grgeom);                                               \
+                                                                                                             \
+              GrGeomOctree  *PV_node;                                                                        \
+              double PV_ref = pow(2.0, r);                                                                   \
+                                                                                                             \
+              i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;                                                 \
+              j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;                                                 \
+              k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;                                                 \
+              GrGeomOctreeExteriorNodeLoop(i, j, k, PV_node,                                                 \
+                                           GrGeomSolidData(grgeom),                                          \
+                                           GrGeomSolidOctreeBGLevel(grgeom) + r,                             \
+                                           ix, iy, iz, nx, ny, nz,                                           \
+                                           TRUE,                                                             \
+        {                                                                                                    \
+          body;                                                                                              \
+          outflag[(k - iz) * ny * nx + (j - iy) * nx + (i - ix)] |= (1 << 1);                                \
+        });                                                                                                  \
+              GrGeomSolidCellFlagInitialized(grgeom) |= (1 << 1);                                            \
+            }                                                                                                \
+            else                                                                                             \
+            {                                                                                                \
+              dim3 block, grid;                                                                              \
+              FindDims(grid, block, nx, ny, nz, 1);                                                          \
+                                                                                                             \
+              char *outflag = GrGeomSolidCellFlagData(grgeom);                                               \
+              auto lambda_body =                                                                             \
+                GPU_LAMBDA(int i, int j, int k)                                                              \
+              {                                                                                              \
+                i += ix;                                                                                     \
+                j += iy;                                                                                     \
+                k += iz;                                                                                     \
+                                                                                                             \
+                if (outflag[(k - iz) * ny * nx + (j - iy) * nx + (i - ix)] & (1 << 1))                       \
+                {                                                                                            \
+                  body;                                                                                      \
+                }                                                                                            \
+              };                                                                                             \
+                                                                                                             \
+              BoxKernel << < grid, block >> > (lambda_body, nx, ny, nz);                                     \
+              CUDA_ERR(cudaPeekAtLastError());                                                               \
+              CUDA_ERR(cudaStreamSynchronize(0));                                                            \
+            }                                                                                                \
+          }                                                                                                  \
+          (void)i; (void)j; (void)k;                                                                         \
+        }
 }
 #endif // PF_CUDALOOPS_H
diff --git a/pfsimulator/parflow_lib/pf_cudamalloc.h b/pfsimulator/parflow_lib/pf_cudamalloc.h
index ea8060b73..eba691eb6 100644
--- a/pfsimulator/parflow_lib/pf_cudamalloc.h
+++ b/pfsimulator/parflow_lib/pf_cudamalloc.h
@@ -37,27 +37,27 @@
 #define tfree_amps_cuda(ptr) amps_TFree_managed(ptr)
 
 #define talloc_cuda(type, count) \
-  ((count) ? (type*)_talloc_device(sizeof(type) * (unsigned int)(count)) : NULL)
+        ((count) ? (type*)_talloc_device(sizeof(type) * (unsigned int)(count)) : NULL)
 
 #define ctalloc_cuda(type, count) \
-  ((count) ? (type*)_ctalloc_device(sizeof(type) * (unsigned int)(count)) : NULL)
+        ((count) ? (type*)_ctalloc_device(sizeof(type) * (unsigned int)(count)) : NULL)
 
 #define tfree_cuda(ptr) if (ptr) _tfree_device(ptr); else {}
 
 #define tmemcpy_cuda(dest, src, bytes) \
-  CUDA_ERR(cudaMemcpy(dest, src, bytes, cudaMemcpyDeviceToDevice))
-
-#define MemPrefetchDeviceToHost_cuda(ptr, size, stream)              \
-{                                                                    \
-  CUDA_ERR(cudaMemPrefetchAsync(ptr, size, cudaCpuDeviceId, stream));\
-  CUDA_ERR(cudaStreamSynchronize(stream));                           \
-}
-
-#define MemPrefetchHostToDevice_cuda(ptr, size, stream)              \
-{                                                                    \
-  int device;                                                        \
-  CUDA_ERR(cudaGetDevice(&device));                                  \
-  CUDA_ERR(cudaMemPrefetchAsync(ptr, size, device, stream))          \
-}
+        CUDA_ERR(cudaMemcpy(dest, src, bytes, cudaMemcpyDeviceToDevice))
+
+#define MemPrefetchDeviceToHost_cuda(ptr, size, stream)                       \
+        {                                                                     \
+          CUDA_ERR(cudaMemPrefetchAsync(ptr, size, cudaCpuDeviceId, stream)); \
+          CUDA_ERR(cudaStreamSynchronize(stream));                            \
+        }
+
+#define MemPrefetchHostToDevice_cuda(ptr, size, stream)                      \
+        {                                                                    \
+          int device;                                                        \
+          CUDA_ERR(cudaGetDevice(&device));                                  \
+          CUDA_ERR(cudaMemPrefetchAsync(ptr, size, device, stream))          \
+        }
 
 #endif // PF_CUDAMALLOC_H
diff --git a/pfsimulator/parflow_lib/pf_devices.h b/pfsimulator/parflow_lib/pf_devices.h
index b4264cdb4..7e4a399ba 100644
--- a/pfsimulator/parflow_lib/pf_devices.h
+++ b/pfsimulator/parflow_lib/pf_devices.h
@@ -37,19 +37,19 @@
 
 /**
  * @brief CUDA error handling.
- * 
+ *
  * If error detected, print error message and exit.
  *
  * @param expr CUDA error (of type cudaError_t) [IN]
  */
-#define CUDA_ERR(expr)                                                                 \
-{                                                                                      \
-  cudaError_t err = expr;                                                              \
-  if (err != cudaSuccess) {                                                            \
-    printf("\n\n%s in %s at line %d\n", cudaGetErrorString(err), __FILE__, __LINE__);  \
-    exit(1);                                                                           \
-  }                                                                                    \
-}
+#define CUDA_ERR(expr)                                                                         \
+        {                                                                                      \
+          cudaError_t err = expr;                                                              \
+          if (err != cudaSuccess) {                                                            \
+            printf("\n\n%s in %s at line %d\n", cudaGetErrorString(err), __FILE__, __LINE__);  \
+            exit(1);                                                                           \
+          }                                                                                    \
+        }
 
 /*--------------------------------------------------------------------------
  * CUDA profiling macros
@@ -57,22 +57,22 @@
 
 /** Record an NVTX range for NSYS if accelerator present. */
 #include "nvToolsExt.h"
-#define PUSH_NVTX_cuda(name,cid)                                                          \
-{                                                                                         \
-  const uint32_t colors_nvtx[] =                                                          \
-    {0xff00ff00, 0xff0000ff, 0xffffff00, 0xffff00ff, 0xff00ffff, 0xffff0000, 0xffffffff}; \
-  const int num_colors_nvtx = sizeof(colors_nvtx)/sizeof(uint32_t);                       \
-  int color_id_nvtx = cid;                                                                \
-  color_id_nvtx = color_id_nvtx%num_colors_nvtx;                                          \
-  nvtxEventAttributes_t eventAttrib = {0};                                                \
-  eventAttrib.version = NVTX_VERSION;                                                     \
-  eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;                                       \
-  eventAttrib.colorType = NVTX_COLOR_ARGB;                                                \
-  eventAttrib.color = colors_nvtx[color_id_nvtx];                                         \
-  eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;                                      \
-  eventAttrib.message.ascii = name;                                                       \
-  nvtxRangePushEx(&eventAttrib);                                                          \
-}
+#define PUSH_NVTX_cuda(name, cid)                                                                   \
+        {                                                                                           \
+          const uint32_t colors_nvtx[] =                                                            \
+          { 0xff00ff00, 0xff0000ff, 0xffffff00, 0xffff00ff, 0xff00ffff, 0xffff0000, 0xffffffff };   \
+          const int num_colors_nvtx = sizeof(colors_nvtx) / sizeof(uint32_t);                       \
+          int color_id_nvtx = cid;                                                                  \
+          color_id_nvtx = color_id_nvtx % num_colors_nvtx;                                          \
+          nvtxEventAttributes_t eventAttrib = { 0 };                                                \
+          eventAttrib.version = NVTX_VERSION;                                                       \
+          eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;                                         \
+          eventAttrib.colorType = NVTX_COLOR_ARGB;                                                  \
+          eventAttrib.color = colors_nvtx[color_id_nvtx];                                           \
+          eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;                                        \
+          eventAttrib.message.ascii = name;                                                         \
+          nvtxRangePushEx(&eventAttrib);                                                            \
+        }
 /** Stop recording an NVTX range for NSYS if accelerator present. */
 #define POP_NVTX_cuda nvtxRangePop();
 
@@ -82,19 +82,19 @@
 #include <rmm/rmm_api.h>
 /**
  * @brief RMM error handling.
- * 
+ *
  * If error detected, print error message and exit.
  *
  * @param expr RMM error (of type rmmError_t) [IN]
  */
-#define RMM_ERR(expr)                                                                  \
-{                                                                                      \
-  rmmError_t err = expr;                                                               \
-  if (err != RMM_SUCCESS) {                                                            \
-    printf("\n\n%s in %s at line %d\n", rmmGetErrorString(err), __FILE__, __LINE__);   \
-    exit(1);                                                                           \
-  }                                                                                    \
-}
+#define RMM_ERR(expr)                                                                          \
+        {                                                                                      \
+          rmmError_t err = expr;                                                               \
+          if (err != RMM_SUCCESS) {                                                            \
+            printf("\n\n%s in %s at line %d\n", rmmGetErrorString(err), __FILE__, __LINE__);   \
+            exit(1);                                                                           \
+          }                                                                                    \
+        }
 #endif
 
 /*--------------------------------------------------------------------------
@@ -123,9 +123,9 @@ void kokkosMemSet(char *ptr, size_t size);
 
 /**
  * @brief Allocates unified memory.
- * 
+ *
  * If RMM library is available, pool allocation is used for better performance.
- * 
+ *
  * @note Should not be called directly.
  *
  * @param size bytes to be allocated [IN]
@@ -134,15 +134,15 @@ void kokkosMemSet(char *ptr, size_t size);
 
 static inline void *_talloc_device(size_t size)
 {
-  void *ptr = NULL;  
-  
+  void *ptr = NULL;
+
 #ifdef PARFLOW_HAVE_RMM
-  RMM_ERR(rmmAlloc(&ptr,size,0,__FILE__,__LINE__));
+  RMM_ERR(rmmAlloc(&ptr, size, 0, __FILE__, __LINE__));
 #elif defined(PARFLOW_HAVE_KOKKOS)
   ptr = kokkosAlloc(size);
 #elif defined(PARFLOW_HAVE_CUDA)
   CUDA_ERR(cudaMallocManaged((void**)&ptr, size, cudaMemAttachGlobal));
-  // CUDA_ERR(cudaHostAlloc((void**)&ptr, size, cudaHostAllocMapped));  
+  // CUDA_ERR(cudaHostAlloc((void**)&ptr, size, cudaHostAllocMapped));
 #endif
 
   return ptr;
@@ -150,9 +150,9 @@ static inline void *_talloc_device(size_t size)
 
 /**
  * @brief Allocates unified memory initialized to 0.
- * 
+ *
  * If RMM library is available, pool allocation is used for better performance.
- * 
+ *
  * @note Should not be called directly.
  *
  * @param size bytes to be allocated [IN]
@@ -160,30 +160,30 @@ static inline void *_talloc_device(size_t size)
  */
 static inline void *_ctalloc_device(size_t size)
 {
-  void *ptr = NULL;  
+  void *ptr = NULL;
 
 #ifdef PARFLOW_HAVE_RMM
-  RMM_ERR(rmmAlloc(&ptr,size,0,__FILE__,__LINE__));
+  RMM_ERR(rmmAlloc(&ptr, size, 0, __FILE__, __LINE__));
 #elif defined(PARFLOW_HAVE_KOKKOS)
   ptr = kokkosAlloc(size);
 #elif defined(PARFLOW_HAVE_CUDA)
   CUDA_ERR(cudaMallocManaged((void**)&ptr, size, cudaMemAttachGlobal));
   // CUDA_ERR(cudaHostAlloc((void**)&ptr, size, cudaHostAllocMapped));
-#endif  
+#endif
 
 #if defined(PARFLOW_HAVE_CUDA)
-  CUDA_ERR(cudaMemset(ptr, 0, size));  
+  CUDA_ERR(cudaMemset(ptr, 0, size));
 #else
   // memset(ptr, 0, size);
   kokkosMemSet((char*)ptr, size);
 #endif
-  
+
   return ptr;
 }
 
 /**
  * @brief Frees unified memory allocated with \ref _talloc_device or \ref _ctalloc_device.
- * 
+ *
  * @note Should not be called directly.
  *
  * @param ptr a void pointer to the allocated dataspace [IN]
@@ -191,7 +191,7 @@ static inline void *_ctalloc_device(size_t size)
 static inline void _tfree_device(void *ptr)
 {
 #ifdef PARFLOW_HAVE_RMM
-  RMM_ERR(rmmFree(ptr,0,__FILE__,__LINE__));
+  RMM_ERR(rmmFree(ptr, 0, __FILE__, __LINE__));
 #elif defined(PARFLOW_HAVE_KOKKOS)
   kokkosFree(ptr);
 #elif defined(PARFLOW_HAVE_CUDA)
diff --git a/pfsimulator/parflow_lib/pf_hypre.c b/pfsimulator/parflow_lib/pf_hypre.c
index 52bd46bc8..62a812337 100644
--- a/pfsimulator/parflow_lib/pf_hypre.c
+++ b/pfsimulator/parflow_lib/pf_hypre.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -35,7 +35,7 @@
 #ifdef HAVE_HYPRE
 #include "hypre_dependences.h"
 
-void CopyParFlowVectorToHypreVector(Vector *rhs,                                    
+void CopyParFlowVectorToHypreVector(Vector *            rhs,
                                     HYPRE_StructVector* hypre_b)
 {
   Grid* grid = VectorGrid(rhs);
@@ -45,7 +45,7 @@ void CopyParFlowVectorToHypreVector(Vector *rhs,
   int nx_v, ny_v, nz_v;
   int i, j, k;
   int index[3];
-    
+
   ForSubgridI(sg, GridSubgrids(grid))
   {
     Subgrid* subgrid = SubgridArraySubgrid(GridSubgrids(grid), sg);
@@ -71,21 +71,19 @@ void CopyParFlowVectorToHypreVector(Vector *rhs,
     BoxLoopI1(i, j, k, ix, iy, iz, nx, ny, nz,
               iv, nx_v, ny_v, nz_v, 1, 1, 1,
     {
-
       index[0] = i;
       index[1] = j;
       index[2] = k;
-      
+
       HYPRE_StructVectorSetValues(*hypre_b, index, rhs_ptr[iv]);
     });
-
   }
   HYPRE_StructVectorAssemble(*hypre_b);
 }
 
 
 void CopyHypreVectorToParflowVector(HYPRE_StructVector* hypre_x,
-                                    Vector *soln)
+                                    Vector *            soln)
 {
   Grid* grid = VectorGrid(soln);
   int sg;
@@ -132,16 +130,16 @@ void CopyHypreVectorToParflowVector(HYPRE_StructVector* hypre_x,
 
 
 void HypreAssembleGrid(
-		       Grid* pf_grid,
-		       HYPRE_StructGrid* hypre_grid,
-		       double* dxyz
-		       )
+                       Grid*             pf_grid,
+                       HYPRE_StructGrid* hypre_grid,
+                       double*           dxyz
+                       )
 {
   int sg;
 
   int ilo[3];
   int ihi[3];
-  
+
   if (pf_grid != NULL)
   {
     /* Free the HYPRE grid */
@@ -175,12 +173,12 @@ void HypreAssembleGrid(
   }
 }
 
-void HypreInitialize(Matrix* pf_Bmat,
-                     HYPRE_StructGrid* hypre_grid,
+void HypreInitialize(Matrix*              pf_Bmat,
+                     HYPRE_StructGrid*    hypre_grid,
                      HYPRE_StructStencil* hypre_stencil,
-                     HYPRE_StructMatrix* hypre_mat,
-                     HYPRE_StructVector* hypre_b,
-                     HYPRE_StructVector* hypre_x
+                     HYPRE_StructMatrix*  hypre_mat,
+                     HYPRE_StructVector*  hypre_b,
+                     HYPRE_StructVector*  hypre_x
                      )
 {
   int full_ghosts[6] = { 1, 1, 1, 1, 1, 1 };
@@ -190,17 +188,18 @@ void HypreInitialize(Matrix* pf_Bmat,
    * entire nonlinear solve process */
   /* Set stencil parameters */
   int stencil_size = MatrixDataStencilSize(pf_Bmat);
+
   if (!(*hypre_stencil))
   {
     HYPRE_StructStencilCreate(3, stencil_size, hypre_stencil);
-    
+
     for (int i = 0; i < stencil_size; i++)
     {
       HYPRE_StructStencilSetElement(*hypre_stencil, i,
-                                      &(MatrixDataStencil(pf_Bmat))[i * 3]);
+                                    &(MatrixDataStencil(pf_Bmat))[i * 3]);
     }
   }
-  
+
   /* Set up new matrix */
   int symmetric = MatrixSymmetric(pf_Bmat);
   if (!(*hypre_mat))
@@ -212,7 +211,7 @@ void HypreInitialize(Matrix* pf_Bmat,
     HYPRE_StructMatrixSetSymmetric(*hypre_mat, symmetric);
     HYPRE_StructMatrixInitialize(*hypre_mat);
   }
-  
+
   /* Set up new right-hand-side vector */
   if (!(*hypre_b))
   {
@@ -222,7 +221,7 @@ void HypreInitialize(Matrix* pf_Bmat,
     HYPRE_StructVectorSetNumGhost(*hypre_b, no_ghosts);
     HYPRE_StructVectorInitialize(*hypre_b);
   }
-  
+
   /* Set up new solution vector */
   if (!(*hypre_x))
   {
@@ -237,10 +236,10 @@ void HypreInitialize(Matrix* pf_Bmat,
 }
 
 void HypreAssembleMatrixAsElements(
-                                   Matrix *     pf_Bmat,
-                                   Matrix *     pf_Cmat,
+                                   Matrix *            pf_Bmat,
+                                   Matrix *            pf_Cmat,
                                    HYPRE_StructMatrix* hypre_mat,
-                                   ProblemData *problem_data
+                                   ProblemData *       problem_data
                                    )
 {
   Grid *mat_grid = MatrixGrid(pf_Bmat);
@@ -264,13 +263,11 @@ void HypreAssembleMatrixAsElements(
   int symmetric = MatrixSymmetric(pf_Bmat);
 
   Vector* top = ProblemDataIndexOfDomainTop(problem_data);
-  
+
   if (pf_Cmat == NULL) /* No overland flow */
   {
     ForSubgridI(sg, GridSubgrids(mat_grid))
     {
-
-      
       Subgrid* subgrid = GridSubgrid(mat_grid, sg);
 
       Submatrix* pfB_sub = MatrixSubmatrix(pf_Bmat, sg);
@@ -294,21 +291,21 @@ void HypreAssembleMatrixAsElements(
         lp = SubmatrixStencilData(pfB_sub, 5);
         up = SubmatrixStencilData(pfB_sub, 6);
       }
-      
+
       ix = SubgridIX(subgrid);
       iy = SubgridIY(subgrid);
       iz = SubgridIZ(subgrid);
-      
+
       nx = SubgridNX(subgrid);
       ny = SubgridNY(subgrid);
       nz = SubgridNZ(subgrid);
-      
+
       nx_m = SubmatrixNX(pfB_sub);
       ny_m = SubmatrixNY(pfB_sub);
       nz_m = SubmatrixNZ(pfB_sub);
-      
+
       im = SubmatrixEltIndex(pfB_sub, ix, iy, iz);
-      
+
       if (symmetric)
       {
         BoxLoopI1(i, j, k, ix, iy, iz, nx, ny, nz,
@@ -356,12 +353,12 @@ void HypreAssembleMatrixAsElements(
     ForSubgridI(sg, GridSubgrids(mat_grid))
     {
       Subgrid* subgrid = GridSubgrid(mat_grid, sg);
-      
+
       Submatrix* pfB_sub = MatrixSubmatrix(pf_Bmat, sg);
       Submatrix* pfC_sub = MatrixSubmatrix(pf_Cmat, sg);
-      
+
       Subvector* top_sub = VectorSubvector(top, sg);
-      
+
       if (symmetric)
       {
         /* Pull off upper diagonal coeffs here for symmetric part */
@@ -369,7 +366,7 @@ void HypreAssembleMatrixAsElements(
         ep = SubmatrixStencilData(pfB_sub, 2);
         np = SubmatrixStencilData(pfB_sub, 4);
         up = SubmatrixStencilData(pfB_sub, 6);
-        
+
         //          cp_c    = SubmatrixStencilData(pfC_sub, 0);
         //          ep_c    = SubmatrixStencilData(pfC_sub, 2);
         //          np_c    = SubmatrixStencilData(pfC_sub, 4);
@@ -389,7 +386,7 @@ void HypreAssembleMatrixAsElements(
         np = SubmatrixStencilData(pfB_sub, 4);
         lp = SubmatrixStencilData(pfB_sub, 5);
         up = SubmatrixStencilData(pfB_sub, 6);
-        
+
         cp_c = SubmatrixStencilData(pfC_sub, 0);
         wp_c = SubmatrixStencilData(pfC_sub, 1);
         ep_c = SubmatrixStencilData(pfC_sub, 2);
@@ -397,23 +394,23 @@ void HypreAssembleMatrixAsElements(
         np_c = SubmatrixStencilData(pfC_sub, 4);
         top_dat = SubvectorData(top_sub);
       }
-      
+
       ix = SubgridIX(subgrid);
       iy = SubgridIY(subgrid);
       iz = SubgridIZ(subgrid);
-      
+
       nx = SubgridNX(subgrid);
       ny = SubgridNY(subgrid);
       nz = SubgridNZ(subgrid);
-      
+
       nx_m = SubmatrixNX(pfB_sub);
       ny_m = SubmatrixNY(pfB_sub);
       nz_m = SubmatrixNZ(pfB_sub);
-      
+
       sy_v = SubvectorNX(top_sub);
-      
+
       im = SubmatrixEltIndex(pfB_sub, ix, iy, iz);
-      
+
       if (symmetric)
       {
         BoxLoopI1(i, j, k, ix, iy, iz, nx, ny, nz,
@@ -442,7 +439,7 @@ void HypreAssembleMatrixAsElements(
             coeffs_symm[2] = np[im];
             coeffs_symm[3] = up[im];
           }
-          
+
           index[0] = i;
           index[1] = j;
           index[2] = k;
@@ -506,7 +503,7 @@ void HypreAssembleMatrixAsElements(
             coeffs[5] = lp[im];
             coeffs[6] = up[im];
           }
-          
+
           index[0] = i;
           index[1] = j;
           index[2] = k;
@@ -518,7 +515,7 @@ void HypreAssembleMatrixAsElements(
       }
     }   /* End subgrid loop */
   }  /* end if pf_Cmat==NULL */
-  
+
   HYPRE_StructMatrixAssemble(*hypre_mat);
 }
 
diff --git a/pfsimulator/parflow_lib/pf_hypre.h b/pfsimulator/parflow_lib/pf_hypre.h
index 031a33d69..96eb75aff 100644
--- a/pfsimulator/parflow_lib/pf_hypre.h
+++ b/pfsimulator/parflow_lib/pf_hypre.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef _PF_HYPRE
 #define _PF_HYPRE
@@ -39,9 +39,9 @@
  * @param hypre_vector the destination vector
  */
 void CopyParFlowVectorToHypreVector(
-				    Vector *pf_vector,				    
-				    HYPRE_StructVector* hypre_vector
-				    );
+                                    Vector *            pf_vector,
+                                    HYPRE_StructVector* hypre_vector
+                                    );
 
 /**
  * Copy a Hypre vector to a ParFlow vector.
@@ -50,23 +50,23 @@ void CopyParFlowVectorToHypreVector(
  * @param pf_vector the destination vector
  */
 void CopyHypreVectorToParflowVector(
-				    HYPRE_StructVector* hypre_vector,
-				    Vector *pf_vector
-				    );
+                                    HYPRE_StructVector* hypre_vector,
+                                    Vector *            pf_vector
+                                    );
 
 
 void HypreAssembleGrid(
-		       Grid* pf_grid,
-		       HYPRE_StructGrid* hypre_grid,
-		       double* dxyz
-		       );
+                       Grid*             pf_grid,
+                       HYPRE_StructGrid* hypre_grid,
+                       double*           dxyz
+                       );
 
 /**
  * Create and initialize Hypre structures.
- * 
+ *
  * Sets up the Hypre data structures for the grid, stencil, matrix and vectors.
  * This setup is common to all the Hypre solvers ParFlow is using so it has
- * been factored out to a common utility function.   Pointers to 
+ * been factored out to a common utility function.   Pointers to
  * the Hypre data structures are returned.
  *
  * @param pf_matrix Source matrix
@@ -76,13 +76,13 @@ void HypreAssembleGrid(
  * @param hypre_rhs Constructed rhs vector
  * @parame hypre_soln Constucted solution vector
  */
-void HypreInitialize(Matrix* pf_matrix,
-		     HYPRE_StructGrid* hypre_grid,
-		     HYPRE_StructStencil* hypre_stencil,
-		     HYPRE_StructMatrix* hypre_mat,
-		     HYPRE_StructVector* hypre_rhs,
-		     HYPRE_StructVector* hypre_soln
-		     );
+void HypreInitialize(Matrix*              pf_matrix,
+                     HYPRE_StructGrid*    hypre_grid,
+                     HYPRE_StructStencil* hypre_stencil,
+                     HYPRE_StructMatrix*  hypre_mat,
+                     HYPRE_StructVector*  hypre_rhs,
+                     HYPRE_StructVector*  hypre_soln
+                     );
 
 /**
  * Assemble the Hypre matrix from B and C ParFlow matrices using an
@@ -102,11 +102,11 @@ void HypreInitialize(Matrix* pf_matrix,
  * @param problem_data ParFlow problem data
  */
 void HypreAssembleMatrixAsElements(
-				   Matrix *     pf_Bmat,
-				   Matrix *     pf_Cmat,
-				   HYPRE_StructMatrix* hypre_mat,
-				   ProblemData *problem_data
-				   );
+                                   Matrix *            pf_Bmat,
+                                   Matrix *            pf_Cmat,
+                                   HYPRE_StructMatrix* hypre_mat,
+                                   ProblemData *       problem_data
+                                   );
 
 #endif
 
diff --git a/pfsimulator/parflow_lib/pf_kokkos.cpp b/pfsimulator/parflow_lib/pf_kokkos.cpp
index 58ec0eb0c..e0ac809ed 100644
--- a/pfsimulator/parflow_lib/pf_kokkos.cpp
+++ b/pfsimulator/parflow_lib/pf_kokkos.cpp
@@ -64,16 +64,14 @@ void kokkosMemSet(char *ptr, size_t size){
  * @brief Initialize Kokkos
  */
 void kokkosInit(){
-  Kokkos::InitArguments args;
-  args.ndevices = 1;
-  Kokkos::initialize(args);    
+  if(!Kokkos::is_initialized()) Kokkos::initialize();
 }
 
 /**
  * @brief Finalize Kokkos
  */
 void kokkosFinalize(){
-  if(Kokkos::is_initialized) Kokkos::finalize();
+  if(Kokkos::is_initialized() && !Kokkos::is_finalized()) Kokkos::finalize();
 }
 
 }
diff --git a/pfsimulator/parflow_lib/pf_kokkosloops.h b/pfsimulator/parflow_lib/pf_kokkosloops.h
index 3777ad79b..f182c8a2c 100644
--- a/pfsimulator/parflow_lib/pf_kokkosloops.h
+++ b/pfsimulator/parflow_lib/pf_kokkosloops.h
@@ -30,8 +30,7 @@
 #include "pf_devices.h"
 #include "pf_kokkosmalloc.h"
 
-extern "C++"{
-
+extern "C++" {
 #include <tuple>
 #include <Kokkos_Core.hpp>
 
@@ -46,37 +45,36 @@ extern "C++"{
 #define RAND48_MULT_2   (0x0005)
 #define RAND48_ADD      (0x000b)
 
- /** Helper struct for type comparison. @note Not for direct use! */
-template <typename T>
+/** Helper struct for type comparison. @note Not for direct use! */
+template < typename T >
 struct function_traits
-    : public function_traits<decltype(&T::operator())>
+  : public function_traits < decltype(&T::operator()) >
 {};
 // For generic types, directly use the result of the signature of its 'operator()'
 
- /** Helper struct for type comparison. @note Not for direct use! */
-template <typename ClassType, typename ReturnType, typename... Args>
-struct function_traits<ReturnType(ClassType::*)(Args...) const>
+/** Helper struct for type comparison. @note Not for direct use! */
+template < typename ClassType, typename ReturnType, typename ... Args >
+struct function_traits < ReturnType (ClassType::*)(Args...) const >
 // we specialize for pointers to member function
 {
-    enum { arity = sizeof...(Args) };
-    // arity is the number of arguments.
-
-    typedef ReturnType result_type;
-
-    template <size_t i>
-    struct arg
-    {
-        typedef typename std::tuple_element<i, std::tuple<Args...>>::type type;
-        // the i-th argument is equivalent to the i-th tuple element of a tuple
-        // composed of those arguments.
-    };
+  enum { arity = sizeof...(Args) };
+  // arity is the number of arguments.
+
+  typedef ReturnType result_type;
+
+  template < size_t i >
+  struct arg {
+    typedef typename std::tuple_element < i, std::tuple < Args... > > ::type type;
+    // the i-th argument is equivalent to the i-th tuple element of a tuple
+    // composed of those arguments.
+  };
 };
 
 /** Device-callable dorand48() function for Kokkos compute kernels. */
 KOKKOS_FORCEINLINE_FUNCTION static void dev_dorand48(unsigned short xseed[3])
 {
   unsigned long accu;
-  
+
   unsigned short _rand48_mult[3] = {
     RAND48_MULT_0,
     RAND48_MULT_1,
@@ -116,11 +114,12 @@ KOKKOS_FORCEINLINE_FUNCTION static double dev_drand48(void)
     RAND48_SEED_1,
     RAND48_SEED_2
   };
+
   return dev_erand48(_rand48_seed);
 }
 
 /** Device-callable RPowerR() function for Kokkos compute kernels. */
-template <typename T>
+template < typename T >
 KOKKOS_FORCEINLINE_FUNCTION static T RPowerR(T base, T exponent)
 {
   if (base <= 0.0)
@@ -129,38 +128,38 @@ KOKKOS_FORCEINLINE_FUNCTION static T RPowerR(T base, T exponent)
   return((T)pow((double)base, (double)exponent));
 }
 
- /** Helper struct for type comparison. @note Not for direct use! */
-struct SkipParallelSync {const int dummy = 0;};
+/** Helper struct for type comparison. @note Not for direct use! */
+struct SkipParallelSync { const int dummy = 0; };
 #define SKIP_PARALLEL_SYNC_kokkos struct SkipParallelSync sync_struct; return sync_struct;
 
-#define PARALLEL_SYNC_kokkos Kokkos::fence(); 
+#define PARALLEL_SYNC_kokkos Kokkos::fence();
 
 #define PlusEquals_kokkos(a, b) Kokkos::atomic_add(&(a), b)
 
- /** Helper struct for type comparison. @note Not for direct use! */
-template <typename T>
-struct ReduceMaxType {T value;};
-#define ReduceMax_kokkos(a, b) if(lsum < b) { lsum = b; } struct ReduceMaxType<std::decay<decltype(a)>::type> reduce_struct; return reduce_struct;
+/** Helper struct for type comparison. @note Not for direct use! */
+template < typename T >
+struct ReduceMaxType { T value; };
+#define ReduceMax_kokkos(a, b) if (lsum < b) { lsum = b; } struct ReduceMaxType < std::decay < decltype(a) > ::type > reduce_struct; return reduce_struct;
 
- /** Helper struct for type comparison. @note Not for direct use! */
-template <typename T>
-struct ReduceMinType {T value;};
-#define ReduceMin_kokkos(a, b) if(lsum > b) { lsum = b; } struct ReduceMinType<std::decay<decltype(a)>::type> reduce_struct; return reduce_struct;
+/** Helper struct for type comparison. @note Not for direct use! */
+template < typename T >
+struct ReduceMinType { T value; };
+#define ReduceMin_kokkos(a, b) if (lsum > b) { lsum = b; } struct ReduceMinType < std::decay < decltype(a) > ::type > reduce_struct; return reduce_struct;
 
- /** Helper struct for type comparison. @note Not for direct use! */
-template <typename T>
-struct ReduceSumType {T value;};
-#define ReduceSum_kokkos(a, b) lsum += b; struct ReduceSumType<std::decay<decltype(a)>::type> reduce_struct; return reduce_struct;
+/** Helper struct for type comparison. @note Not for direct use! */
+template < typename T >
+struct ReduceSumType { T value; };
+#define ReduceSum_kokkos(a, b) lsum += b; struct ReduceSumType < std::decay < decltype(a) > ::type > reduce_struct; return reduce_struct;
 
 
 /** A constant table for fdir (borrowed from OMP backend) */
 static const int FDIR_TABLE[6][3] = {
-  {-1, 0,  0}, // Left
-  {1,  0,  0}, // Right
-  {0, -1,  0}, // Down
-  {0,  1,  0}, // Up
-  {0,  0, -1}, // Back
-  {0,  0,  1}, // Front
+  { -1, 0, 0 }, // Left
+  { 1, 0, 0 }, // Right
+  { 0, -1, 0 }, // Down
+  { 0, 1, 0 }, // Up
+  { 0, 0, -1 }, // Back
+  { 0, 0, 1 }, // Front
 };
 
 /*--------------------------------------------------------------------------
@@ -177,760 +176,759 @@ static const int FDIR_TABLE[6][3] = {
  * @param ny The size of the second dim [IN]
  * @param nz The size of the third dim [IN]
  */
-#define CheckCellFlagAllocation(grgeom, nx, ny, nz)                                 \
-{                                                                                   \
-  int flagdata_size = sizeof(char) * (nz * ny * nx);                                \
-  if(GrGeomSolidCellFlagDataSize(grgeom) < flagdata_size)                           \
-  {                                                                                 \
-    char *flagdata = (char*)_ctalloc_device(flagdata_size);                         \
-                                                                                    \
-    if(GrGeomSolidCellFlagDataSize(grgeom) > 0)                                     \
-      memcpy(flagdata, GrGeomSolidCellFlagData(grgeom),                             \
-        GrGeomSolidCellFlagDataSize(grgeom));                                       \
-                                                                                    \
-    _tfree_device(GrGeomSolidCellFlagData(grgeom));                                 \
-    GrGeomSolidCellFlagData(grgeom) = flagdata;                                     \
-    GrGeomSolidCellFlagDataSize(grgeom) = flagdata_size;                            \
-  }                                                                                 \
-}
-
- /** Loop definition for Kokkos. */
-#define BoxLoopI1_kokkos(i, j, k,                                                   \
-  ix, iy, iz, nx, ny, nz,                                                           \
-  i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                 \
-  loop_body)                                                                        \
-{                                                                                   \
-  if(nx > 0 && ny > 0 && nz > 0)                                                    \
-  {                                                                                 \
-    DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);     \
-                                                                                    \
-    const auto &ref_i1 = i1;                                                        \
-                                                                                    \
-    auto lambda_body =                                                              \
-      KOKKOS_LAMBDA(int i, int j, int k)                                            \
-      {                                                                             \
-        const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                     \
-          + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                              \
-                                                                                    \
-        i += ix;                                                                    \
-        j += iy;                                                                    \
-        k += iz;                                                                    \
-                                                                                    \
-        loop_body;                                                                  \
-      };                                                                            \
-                                                                                    \
-    using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
-    MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx, ny, nz}});                       \
-    Kokkos::parallel_for(mdpolicy_3d, lambda_body);                                 \
-                                                                                    \
-    typedef function_traits<decltype(lambda_body)> traits;                          \
-    if(!std::is_same<traits::result_type, struct SkipParallelSync>::value)          \
-      Kokkos::fence();                                                              \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
-
- /** Loop definition for Kokkos. */
-#define BoxLoopI2_kokkos(i, j, k,                                                   \
-  ix, iy, iz, nx, ny, nz,                                                           \
-  i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                 \
-  i2, nx2, ny2, nz2, sx2, sy2, sz2,                                                 \
-  loop_body)                                                                        \
-{                                                                                   \
-  if(nx > 0 && ny > 0 && nz > 0)                                                    \
-  {                                                                                 \
-    DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);     \
-    DeclareInc(PV_jinc_2, PV_kinc_2, nx, ny, nz, nx2, ny2, nz2, sx2, sy2, sz2);     \
-                                                                                    \
-    const auto &ref_i1 = i1;                                                        \
-    const auto &ref_i2 = i2;                                                        \
-                                                                                    \
-    auto lambda_body =                                                              \
-      KOKKOS_LAMBDA(int i, int j, int k)                                            \
-      {                                                                             \
-        const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                     \
-          + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                              \
-        const int i2 = k * PV_kinc_2 + (k * ny + j) * PV_jinc_2                     \
-          + (k * ny * nx + j * nx + i) * sx2 + ref_i2;                              \
-                                                                                    \
-        i += ix;                                                                    \
-        j += iy;                                                                    \
-        k += iz;                                                                    \
-                                                                                    \
-        loop_body;                                                                  \
-      };                                                                            \
-                                                                                    \
-    using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
-    MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx, ny, nz}});                       \
-    Kokkos::parallel_for(mdpolicy_3d, lambda_body);                                 \
-                                                                                    \
-    typedef function_traits<decltype(lambda_body)> traits;                          \
-    if(!std::is_same<traits::result_type, struct SkipParallelSync>::value)          \
-      Kokkos::fence();                                                              \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
-
- /** Loop definition for Kokkos. */
-#define BoxLoopI3_kokkos(i, j, k,                                                   \
-  ix, iy, iz, nx, ny, nz,                                                           \
-  i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                 \
-  i2, nx2, ny2, nz2, sx2, sy2, sz2,                                                 \
-  i3, nx3, ny3, nz3, sx3, sy3, sz3,                                                 \
-  loop_body)                                                                        \
-{                                                                                   \
-  if(nx > 0 && ny > 0 && nz > 0)                                                    \
-  {                                                                                 \
-    DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);     \
-    DeclareInc(PV_jinc_2, PV_kinc_2, nx, ny, nz, nx2, ny2, nz2, sx2, sy2, sz2);     \
-    DeclareInc(PV_jinc_3, PV_kinc_3, nx, ny, nz, nx3, ny3, nz3, sx3, sy3, sz3);     \
-                                                                                    \
-    const auto &ref_i1 = i1;                                                        \
-    const auto &ref_i2 = i2;                                                        \
-    const auto &ref_i3 = i3;                                                        \
-                                                                                    \
-    auto lambda_body =                                                              \
-      KOKKOS_LAMBDA(int i, int j, int k)                                            \
-      {                                                                             \
-        const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                     \
-          + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                              \
-        const int i2 = k * PV_kinc_2 + (k * ny + j) * PV_jinc_2                     \
-          + (k * ny * nx + j * nx + i) * sx2 + ref_i2;                              \
-        const int i3 = k * PV_kinc_3 + (k * ny + j) * PV_jinc_3                     \
-          + (k * ny * nx + j * nx + i) * sx3 + ref_i3;                              \
-                                                                                    \
-        i += ix;                                                                    \
-        j += iy;                                                                    \
-        k += iz;                                                                    \
-                                                                                    \
-        loop_body;                                                                  \
-      };                                                                            \
-                                                                                    \
-    using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
-    MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx, ny, nz}});                       \
-    Kokkos::parallel_for(mdpolicy_3d, lambda_body);                                 \
-                                                                                    \
-    typedef function_traits<decltype(lambda_body)> traits;                          \
-    if(!std::is_same<traits::result_type, struct SkipParallelSync>::value)          \
-      Kokkos::fence();                                                              \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
-
- /** Loop definition for Kokkos. */
-#define BoxLoopReduceI1_kokkos(rslt, i, j, k,                                       \
-  ix, iy, iz, nx, ny, nz,                                                           \
-  i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                 \
-  loop_body)                                                                        \
-{                                                                                   \
-  if(nx > 0 && ny > 0 && nz > 0)                                                    \
-  {                                                                                 \
-    DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);     \
-                                                                                    \
-    const auto &ref_rslt = rslt;                                                    \
-    const auto &ref_i1 = i1;                                                        \
-                                                                                    \
-    auto lambda_body =                                                              \
-      KOKKOS_LAMBDA(int i, int j, int k, double& lsum)                              \
-      {                                                                             \
-        const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                     \
-          + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                              \
-                                                                                    \
-        i += ix;                                                                    \
-        j += iy;                                                                    \
-        k += iz;                                                                    \
-                                                                                    \
-        loop_body;                                                                  \
-      };                                                                            \
-                                                                                    \
-    auto lambda_outer =                                                             \
-      KOKKOS_LAMBDA(int i, int j, int k, double& lsum)                              \
-      {                                                                             \
-        lambda_body(i, j, k, lsum);                                                 \
-      };                                                                            \
-                                                                                    \
-    using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
-    MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx, ny, nz}});                       \
-    typedef function_traits<decltype(lambda_body)> traits;                          \
-    if(std::is_same<traits::result_type, struct ReduceSumType<double>>::value)      \
-    {                                                                               \
-      Kokkos::parallel_reduce(mdpolicy_3d, lambda_outer, rslt);                     \
-    }                                                                               \
-    else if(std::is_same<traits::result_type, struct ReduceMaxType<double>>::value) \
-    {                                                                               \
-      Kokkos::parallel_reduce(mdpolicy_3d, lambda_outer, Kokkos::Max<double>(rslt));\
-    }                                                                               \
-    else if(std::is_same<traits::result_type, struct ReduceMinType<double>>::value) \
-    {                                                                               \
-      Kokkos::parallel_reduce(mdpolicy_3d, lambda_outer, Kokkos::Min<double>(rslt));\
-    }                                                                               \
-    else                                                                            \
-    {                                                                               \
-      printf("ERROR at %s:%d: Invalid reduction identifier,                         \
-      likely a problem with a BoxLoopReduce body.", __FILE__, __LINE__);            \
-    }                                                                               \
-    Kokkos::fence();                                                                \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
-
- /** Loop definition for Kokkos. */
-#define BoxLoopReduceI2_kokkos(rslt, i, j, k,                                       \
-  ix, iy, iz, nx, ny, nz,                                                           \
-  i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                 \
-  i2, nx2, ny2, nz2, sx2, sy2, sz2,                                                 \
-  loop_body)                                                                        \
-{                                                                                   \
-  if(nx > 0 && ny > 0 && nz > 0)                                                    \
-  {                                                                                 \
-    DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);     \
-    DeclareInc(PV_jinc_2, PV_kinc_2, nx, ny, nz, nx2, ny2, nz2, sx2, sy2, sz2);     \
-                                                                                    \
-    const auto &ref_rslt = rslt;                                                    \
-    const auto &ref_i1 = i1;                                                        \
-    const auto &ref_i2 = i2;                                                        \
-                                                                                    \
-    auto lambda_body =                                                              \
-      KOKKOS_LAMBDA(int i, int j, int k, double& lsum)                              \
-      {                                                                             \
-        const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                     \
-          + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                              \
-        const int i2 = k * PV_kinc_2 + (k * ny + j) * PV_jinc_2                     \
-          + (k * ny * nx + j * nx + i) * sx2 + ref_i2;                              \
-                                                                                    \
-        i += ix;                                                                    \
-        j += iy;                                                                    \
-        k += iz;                                                                    \
-                                                                                    \
-        loop_body;                                                                  \
-      };                                                                            \
-                                                                                    \
-    auto lambda_outer =                                                             \
-      KOKKOS_LAMBDA(int i, int j, int k, double& lsum)                              \
-      {                                                                             \
-        lambda_body(i, j, k, lsum);                                                 \
-      };                                                                            \
-                                                                                    \
-    using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
-    MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx, ny, nz}});                       \
-    typedef function_traits<decltype(lambda_body)> traits;                          \
-    if(std::is_same<traits::result_type, struct ReduceSumType<double>>::value)      \
-    {                                                                               \
-      Kokkos::parallel_reduce(mdpolicy_3d, lambda_outer, rslt);                     \
-    }                                                                               \
-    else if(std::is_same<traits::result_type, struct ReduceMaxType<double>>::value) \
-    {                                                                               \
-      Kokkos::parallel_reduce(mdpolicy_3d, lambda_outer, Kokkos::Max<double>(rslt));\
-    }                                                                               \
-    else if(std::is_same<traits::result_type, struct ReduceMinType<double>>::value) \
-    {                                                                               \
-      Kokkos::parallel_reduce(mdpolicy_3d, lambda_outer, Kokkos::Min<double>(rslt));\
-    }                                                                               \
-    else                                                                            \
-    {                                                                               \
-      printf("ERROR at %s:%d: Invalid reduction identifier,                         \
-      likely a problem with a BoxLoopReduce body.", __FILE__, __LINE__);            \
-    }                                                                               \
-    Kokkos::fence();                                                                \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
-
- /** Loop definition for Kokkos. */
-#define GrGeomInLoopBoxes_kokkos(i, j, k,                                           \
-  grgeom, ix, iy, iz, nx, ny, nz, loop_body)                                        \
-{                                                                                   \
-  BoxArray* boxes = GrGeomSolidInteriorBoxes(grgeom);                               \
-  int ix_bxs = BoxArrayMinCell(boxes, 0);                                           \
-  int iy_bxs = BoxArrayMinCell(boxes, 1);                                           \
-  int iz_bxs = BoxArrayMinCell(boxes, 2);                                           \
-                                                                                    \
-  int nx_bxs = BoxArrayMaxCell(boxes, 0) - ix_bxs + 1;                              \
-  int ny_bxs = BoxArrayMaxCell(boxes, 1) - iy_bxs + 1;                              \
-  int nz_bxs = BoxArrayMaxCell(boxes, 2) - iz_bxs + 1;                              \
-                                                                                    \
-  if(!(GrGeomSolidCellFlagInitialized(grgeom) & 1))                                 \
-  {                                                                                 \
-    CheckCellFlagAllocation(grgeom, nx_bxs, ny_bxs, nz_bxs);                        \
-    char *inflag = GrGeomSolidCellFlagData(grgeom);                                 \
-                                                                                    \
-    for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                    \
-    {                                                                               \
-      Box box = BoxArrayGetBox(boxes, PV_box);                                      \
-      int PV_ixl = box.lo[0];                                                       \
-      int PV_iyl = box.lo[1];                                                       \
-      int PV_izl = box.lo[2];                                                       \
-      int PV_ixu = box.up[0];                                                       \
-      int PV_iyu = box.up[1];                                                       \
-      int PV_izu = box.up[2];                                                       \
-                                                                                    \
-      if(PV_ixl <= PV_ixu && PV_iyl <= PV_iyu && PV_izl <= PV_izu)                  \
-      {                                                                             \
-        int PV_nx = PV_ixu - PV_ixl + 1;                                            \
-        int PV_ny = PV_iyu - PV_iyl + 1;                                            \
-        int PV_nz = PV_izu - PV_izl + 1;                                            \
-                                                                                    \
-        Globals *globals = ::globals;                                               \
-        auto lambda_body =                                                          \
-          KOKKOS_LAMBDA(int i, int j, int k)                                        \
-          {                                                                         \
-            i += PV_ixl;                                                            \
-            j += PV_iyl;                                                            \
-            k += PV_izl;                                                            \
-                                                                                    \
-            /* Set inflag for all cells in boxes regardless of loop limits */       \
-            inflag[(k - iz_bxs) * ny_bxs * nx_bxs +                                 \
-              (j - iy_bxs) * nx_bxs + (i - ix_bxs)] |= 1;                           \
-                                                                                    \
-            /* Only evaluate loop body if the cell is within loop limits */         \
-            if(i >= ix && j >= iy && k >= iz &&                                     \
-              i < ix + nx && j < iy + ny && k < iz + nz)                            \
-            {                                                                       \
-              loop_body;                                                            \
-            }                                                                       \
-          };                                                                        \
-                                                                                    \
-        using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
-        MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{PV_nx, PV_ny, PV_nz}});          \
-        Kokkos::parallel_for(mdpolicy_3d, lambda_body);                             \
-        Kokkos::fence();                                                            \
-      }                                                                             \
-    }                                                                               \
-    GrGeomSolidCellFlagInitialized(grgeom) |= 1;                                    \
-  }                                                                                 \
-  else                                                                              \
-  {                                                                                 \
-    int ixl_gpu = pfmax(ix, ix_bxs);                                                \
-    int iyl_gpu = pfmax(iy, iy_bxs);                                                \
-    int izl_gpu = pfmax(iz, iz_bxs);                                                \
-    int nx_gpu = pfmin((ix + nx - 1), BoxArrayMaxCell(boxes, 0)) - ixl_gpu + 1;     \
-    int ny_gpu = pfmin((iy + ny - 1), BoxArrayMaxCell(boxes, 1)) - iyl_gpu + 1;     \
-    int nz_gpu = pfmin((iz + nz - 1), BoxArrayMaxCell(boxes, 2)) - izl_gpu + 1;     \
-                                                                                    \
-    if(nx_gpu > 0 && ny_gpu > 0 && nz_gpu > 0)                                      \
-    {                                                                               \
-      Globals *globals = ::globals;                                                 \
-      char *inflag = GrGeomSolidCellFlagData(grgeom);                               \
-      auto lambda_body =                                                            \
-        KOKKOS_LAMBDA(int i, int j, int k)                                          \
-        {                                                                           \
-          i += ixl_gpu;                                                             \
-          j += iyl_gpu;                                                             \
-          k += izl_gpu;                                                             \
-          if(inflag[(k - iz_bxs) * ny_bxs * nx_bxs +                                \
-            (j - iy_bxs) * nx_bxs + (i - ix_bxs)] & 1)                              \
-          {                                                                         \
-            loop_body;                                                              \
-          }                                                                         \
-        };                                                                          \
-                                                                                    \
-      using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
-      MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx_gpu, ny_gpu, nz_gpu}});         \
-      Kokkos::parallel_for(mdpolicy_3d, lambda_body);                               \
-      Kokkos::fence();                                                              \
-    }                                                                               \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
-
- /** Loop definition for Kokkos. */
-#define GrGeomSurfLoopBoxes_kokkos(i, j, k, fdir, grgeom,                           \
-  ix, iy, iz, nx, ny, nz, loop_body)                                                \
-{                                                                                   \
-  for (int PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                           \
-  {                                                                                 \
-    const int *fdir = FDIR_TABLE[PV_f];                                             \
-                                                                                    \
-    BoxArray* boxes = GrGeomSolidSurfaceBoxes(grgeom, PV_f);                        \
-    int ix_bxs = BoxArrayMinCell(boxes, 0);                                         \
-    int iy_bxs = BoxArrayMinCell(boxes, 1);                                         \
-    int iz_bxs = BoxArrayMinCell(boxes, 2);                                         \
-                                                                                    \
-    int nx_bxs = BoxArrayMaxCell(boxes, 0) - ix_bxs + 1;                            \
-    int ny_bxs = BoxArrayMaxCell(boxes, 1) - iy_bxs + 1;                            \
-    int nz_bxs = BoxArrayMaxCell(boxes, 2) - iz_bxs + 1;                            \
-                                                                                    \
-    if(!(GrGeomSolidCellFlagInitialized(grgeom) & (1 << (2 + PV_f))))               \
-    {                                                                               \
-      CheckCellFlagAllocation(grgeom, nx_bxs, ny_bxs, nz_bxs);                      \
-      char *surfflag = GrGeomSolidCellFlagData(grgeom);                             \
-                                                                                    \
-      for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                  \
-      {                                                                             \
-        Box box = BoxArrayGetBox(boxes, PV_box);                                    \
-        int PV_ixl = box.lo[0];                                                     \
-        int PV_iyl = box.lo[1];                                                     \
-        int PV_izl = box.lo[2];                                                     \
-        int PV_ixu = box.up[0];                                                     \
-        int PV_iyu = box.up[1];                                                     \
-        int PV_izu = box.up[2];                                                     \
-                                                                                    \
-        if(PV_ixl <= PV_ixu && PV_iyl <= PV_iyu && PV_izl <= PV_izu)                \
-        {                                                                           \
-          int PV_nx = PV_ixu - PV_ixl + 1;                                          \
-          int PV_ny = PV_iyu - PV_iyl + 1;                                          \
-          int PV_nz = PV_izu - PV_izl + 1;                                          \
-                                                                                    \
-          const int _fdir0 = fdir[0];                                               \
-          const int _fdir1 = fdir[1];                                               \
-          const int _fdir2 = fdir[2];                                               \
-                                                                                    \
-          auto lambda_body =                                                        \
-            KOKKOS_LAMBDA(int i, int j, int k)                                      \
-            {                                                                       \
-              i += PV_ixl;                                                          \
-              j += PV_iyl;                                                          \
-              k += PV_izl;                                                          \
-                                                                                    \
-              /* Set surfflag for all cells in boxes regardless of loop limits */   \
-              surfflag[(k - iz_bxs) * ny_bxs * nx_bxs +                             \
-                (j - iy_bxs) * nx_bxs + (i - ix_bxs)] |= (1 << (2 + PV_f));         \
-                                                                                    \
-              /* Only evaluate loop body if the cell is within loop limits */       \
-              if(i >= ix && j >= iy && k >= iz &&                                   \
-                i < ix + nx && j < iy + ny && k < iz + nz)                          \
-              {                                                                     \
-                const int fdir[3] = {_fdir0, _fdir1, _fdir2};                       \
-                loop_body;                                                          \
-                (void)fdir;                                                         \
-              }                                                                     \
-            };                                                                      \
-                                                                                    \
-          using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
-          MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{PV_nx, PV_ny, PV_nz}});        \
-          Kokkos::parallel_for(mdpolicy_3d, lambda_body);                           \
-          Kokkos::fence();                                                          \
-        }                                                                           \
-      }                                                                             \
-      GrGeomSolidCellFlagInitialized(grgeom) |= (1 << (2 + PV_f));                  \
-    }                                                                               \
-    else                                                                            \
-    {                                                                               \
-      int ixl_gpu = pfmax(ix, ix_bxs);                                              \
-      int iyl_gpu = pfmax(iy, iy_bxs);                                              \
-      int izl_gpu = pfmax(iz, iz_bxs);                                              \
-      int nx_gpu = pfmin((ix + nx - 1), BoxArrayMaxCell(boxes, 0)) - ixl_gpu + 1;   \
-      int ny_gpu = pfmin((iy + ny - 1), BoxArrayMaxCell(boxes, 1)) - iyl_gpu + 1;   \
-      int nz_gpu = pfmin((iz + nz - 1), BoxArrayMaxCell(boxes, 2)) - izl_gpu + 1;   \
-                                                                                    \
-      if(nx_gpu > 0 && ny_gpu > 0 && nz_gpu > 0)                                    \
-      {                                                                             \
-        char *surfflag = GrGeomSolidCellFlagData(grgeom);                           \
-                                                                                    \
-        const int _fdir0 = fdir[0];                                                 \
-        const int _fdir1 = fdir[1];                                                 \
-        const int _fdir2 = fdir[2];                                                 \
-                                                                                    \
-        auto lambda_body =                                                          \
-          KOKKOS_LAMBDA(int i, int j, int k)                                        \
-          {                                                                         \
-            i += ixl_gpu;                                                           \
-            j += iyl_gpu;                                                           \
-            k += izl_gpu;                                                           \
-                                                                                    \
-            if(surfflag[(k - iz_bxs) * ny_bxs * nx_bxs +                            \
-              (j - iy_bxs) * nx_bxs + (i - ix_bxs)] & (1 << (2 + PV_f)))            \
-            {                                                                       \
-              const int fdir[3] = {_fdir0, _fdir1, _fdir2};                         \
-              loop_body;                                                            \
-              (void)fdir;                                                           \
-            }                                                                       \
-          };                                                                        \
-                                                                                    \
-        using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
-        MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx_gpu, ny_gpu, nz_gpu}});       \
-        Kokkos::parallel_for(mdpolicy_3d, lambda_body);                             \
-        Kokkos::fence();                                                            \
-      }                                                                             \
-    }                                                                               \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
-
- /** Loop definition for Kokkos. */
-#define GrGeomPatchLoopBoxes_kokkos(i, j, k, fdir, grgeom, patch_num,               \
-  ix, iy, iz, nx, ny, nz, loop_body)                                                \
-{                                                                                   \
-  for (int PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                           \
-  {                                                                                 \
-    const int *fdir = FDIR_TABLE[PV_f];                                             \
-                                                                                    \
-    int n_prev = 0;                                                                 \
-    BoxArray* boxes = GrGeomSolidPatchBoxes(grgeom, patch_num, PV_f);               \
-    for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                    \
-    {                                                                               \
-      Box box = BoxArrayGetBox(boxes, PV_box);                                      \
-      /* find octree and region intersection */                                     \
-      int PV_ixl = pfmax(ix, box.lo[0]);                                            \
-      int PV_iyl = pfmax(iy, box.lo[1]);                                            \
-      int PV_izl = pfmax(iz, box.lo[2]);                                            \
-      int PV_ixu = pfmin((ix + nx - 1), box.up[0]);                                 \
-      int PV_iyu = pfmin((iy + ny - 1), box.up[1]);                                 \
-      int PV_izu = pfmin((iz + nz - 1), box.up[2]);                                 \
-                                                                                    \
-      if(PV_ixl <= PV_ixu && PV_iyl <= PV_iyu && PV_izl <= PV_izu)                  \
-      {                                                                             \
-        int PV_diff_x = PV_ixu - PV_ixl;                                            \
-        int PV_diff_y = PV_iyu - PV_iyl;                                            \
-        int PV_diff_z = PV_izu - PV_izl;                                            \
-                                                                                    \
-        int nx = PV_diff_x + 1;                                                     \
-        int ny = PV_diff_y + 1;                                                     \
-        int nz = PV_diff_z + 1;                                                     \
-                                                                                    \
-        const int _fdir0 = fdir[0];                                                 \
-        const int _fdir1 = fdir[1];                                                 \
-        const int _fdir2 = fdir[2];                                                 \
-                                                                                    \
-        auto lambda_body =                                                          \
-          KOKKOS_LAMBDA(int i, int j, int k)                                        \
-          {                                                                         \
-            const int fdir[3] = {_fdir0, _fdir1, _fdir2};                           \
-            int ival = n_prev + k * ny * nx + j * nx + i;                           \
-                                                                                    \
-            i += PV_ixl;                                                            \
-            j += PV_iyl;                                                            \
-            k += PV_izl;                                                            \
-                                                                                    \
-            loop_body;                                                              \
-            (void)fdir;                                                             \
-          };                                                                        \
-        n_prev += nz * ny * nx;                                                     \
-                                                                                    \
-        using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
-        MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx, ny, nz}});                   \
-        Kokkos::parallel_for(mdpolicy_3d, lambda_body);                             \
-        Kokkos::fence();                                                            \
-      }                                                                             \
-    }                                                                               \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
-
- /** Loop definition for Kokkos. */
-#define GrGeomPatchLoopBoxesNoFdir_kokkos(i, j, k, grgeom, patch_num, ovrlnd,       \
-  ix, iy, iz, nx, ny, nz, locals, setup,                                            \
-  f_left, f_right, f_down, f_up, f_back, f_front, finalize)                         \
-{                                                                                   \
-  int n_ival = 0;                                                                   \
-  for (int PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                           \
-  {                                                                                 \
-    BoxArray* boxes = GrGeomSolidPatchBoxes(grgeom, patch_num, PV_f);               \
-                                                                                    \
-    int ix_bxs = BoxArrayMinCell(boxes, 0);                                         \
-    int iy_bxs = BoxArrayMinCell(boxes, 1);                                         \
-    int iz_bxs = BoxArrayMinCell(boxes, 2);                                         \
-                                                                                    \
-    int nx_bxs = BoxArrayMaxCell(boxes, 0) - ix_bxs + 1;                            \
-    int ny_bxs = BoxArrayMaxCell(boxes, 1) - iy_bxs + 1;                            \
-    int nz_bxs = BoxArrayMaxCell(boxes, 2) - iz_bxs + 1;                            \
-                                                                                    \
-    int patch_loc;                                                                  \
-    if(ovrlnd)                                                                      \
-      patch_loc = GrGeomSolidNumPatches(grgeom) + patch_num;                        \
-    else                                                                            \
-      patch_loc = patch_num;                                                        \
-                                                                                    \
-    int *ptr_ival = GrGeomSolidCellIval(grgeom, patch_loc, PV_f);                   \
-    if(!(ptr_ival))                                                                 \
-    {                                                                               \
-      GrGeomSolidCellIval(grgeom, patch_loc, PV_f) =                                \
-        (int*)_talloc_device(sizeof(int) * nx_bxs * ny_bxs * nz_bxs);               \
-                                                                                    \
-      ptr_ival = GrGeomSolidCellIval(grgeom, patch_loc, PV_f);                      \
-      for (int idx = 0; idx < nx_bxs * ny_bxs * nz_bxs; idx++)                      \
-        ptr_ival[idx] = -1;                                                         \
-                                                                                    \
-      for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                  \
-      {                                                                             \
-        Box box = BoxArrayGetBox(boxes, PV_box);                                    \
-        int PV_ixl = pfmax(ix, box.lo[0]);                                          \
-        int PV_iyl = pfmax(iy, box.lo[1]);                                          \
-        int PV_izl = pfmax(iz, box.lo[2]);                                          \
-        int PV_ixu = pfmin((ix + nx - 1), box.up[0]);                               \
-        int PV_iyu = pfmin((iy + ny - 1), box.up[1]);                               \
-        int PV_izu = pfmin((iz + nz - 1), box.up[2]);                               \
-                                                                                    \
-        for (k = PV_izl; k <= PV_izu; k++)                                          \
-          for (j = PV_iyl; j <= PV_iyu; j++)                                        \
-            for (i = PV_ixl; i <= PV_ixu; i++)                                      \
-            {                                                                       \
-              UNPACK(locals);                                                       \
-              setup;                                                                \
-              switch(PV_f)                                                          \
-              {                                                                     \
-                f_left;                                                             \
-                f_right;                                                            \
-                f_down;                                                             \
-                f_up;                                                               \
-                f_back;                                                             \
-                f_front;                                                            \
-              }                                                                     \
-              finalize;                                                             \
-              ptr_ival[(k - iz_bxs) * ny_bxs * nx_bxs + (j - iy_bxs) *              \
-                nx_bxs + (i - ix_bxs)] = n_ival++;                                  \
-            }                                                                       \
-      }                                                                             \
-    }                                                                               \
-    else                                                                            \
-    {                                                                               \
-      int ixl_gpu = pfmax(ix, ix_bxs);                                              \
-      int iyl_gpu = pfmax(iy, iy_bxs);                                              \
-      int izl_gpu = pfmax(iz, iz_bxs);                                              \
-      int nx_gpu = pfmin((ix + nx - 1), BoxArrayMaxCell(boxes, 0)) - ixl_gpu + 1;   \
-      int ny_gpu = pfmin((iy + ny - 1), BoxArrayMaxCell(boxes, 1)) - iyl_gpu + 1;   \
-      int nz_gpu = pfmin((iz + nz - 1), BoxArrayMaxCell(boxes, 2)) - izl_gpu + 1;   \
-                                                                                    \
-      if(nx_gpu > 0 && ny_gpu > 0 && nz_gpu > 0)                                    \
-      {                                                                             \
-        auto lambda_body =                                                          \
-          KOKKOS_LAMBDA(int i, int j, int k)                                        \
-          {                                                                         \
-            i += ixl_gpu;                                                           \
-            j += iyl_gpu;                                                           \
-            k += izl_gpu;                                                           \
-                                                                                    \
-            int ival = ptr_ival[(k - iz_bxs) * ny_bxs * nx_bxs +                    \
-              (j - iy_bxs) * nx_bxs + (i - ix_bxs)];                                \
-            if(ival >= 0)                                                           \
-            {                                                                       \
-              UNPACK(locals);                                                       \
-              setup;                                                                \
-              switch(PV_f)                                                          \
-              {                                                                     \
-                f_left;                                                             \
-                f_right;                                                            \
-                f_down;                                                             \
-                f_up;                                                               \
-                f_back;                                                             \
-                f_front;                                                            \
-              }                                                                     \
-              finalize;                                                             \
-            }                                                                       \
-          };                                                                        \
-                                                                                    \
-        using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
-        MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx_gpu, ny_gpu, nz_gpu}});       \
-        Kokkos::parallel_for(mdpolicy_3d, lambda_body);                             \
-        Kokkos::fence();                                                            \
-      }                                                                             \
-    }                                                                               \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
-
- /** Loop definition for Kokkos. */
-#define GrGeomOctreeExteriorNodeLoop_kokkos(i, j, k, node, octree, level,           \
-  ix, iy, iz, nx, ny, nz, val_test, loop_body)                                      \
-{                                                                                   \
-  int PV_i, PV_j, PV_k, PV_l;                                                       \
-  int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                               \
-                                                                                    \
-  PV_i = i;                                                                         \
-  PV_j = j;                                                                         \
-  PV_k = k;                                                                         \
-                                                                                    \
-  GrGeomOctreeExteriorLoop(PV_i, PV_j, PV_k, PV_l, node, octree, level, val_test,   \
-  {                                                                                 \
-    if ((PV_i >= ix) && (PV_i < (ix + nx)) &&                                       \
-        (PV_j >= iy) && (PV_j < (iy + ny)) &&                                       \
-        (PV_k >= iz) && (PV_k < (iz + nz)))                                         \
-    {                                                                               \
-      i = PV_i;                                                                     \
-      j = PV_j;                                                                     \
-      k = PV_k;                                                                     \
-      loop_body;                                                                    \
-    }                                                                               \
-  },                                                                                \
-  {                                                                                 \
-    /* find octree and region intersection */                                       \
-    PV_ixl = pfmax(ix, PV_i);                                                       \
-    PV_iyl = pfmax(iy, PV_j);                                                       \
-    PV_izl = pfmax(iz, PV_k);                                                       \
-    PV_ixu = pfmin((ix + nx), (PV_i + (int)PV_inc));                                \
-    PV_iyu = pfmin((iy + ny), (PV_j + (int)PV_inc));                                \
-    PV_izu = pfmin((iz + nz), (PV_k + (int)PV_inc));                                \
-                                                                                    \
-    if(PV_ixl < PV_ixu && PV_iyl < PV_iyu && PV_izl < PV_izu)                       \
-    {                                                                               \
-      const int PV_diff_x = PV_ixu - PV_ixl;                                        \
-      const int PV_diff_y = PV_iyu - PV_iyl;                                        \
-      const int PV_diff_z = PV_izu - PV_izl;                                        \
-                                                                                    \
-      auto lambda_body =                                                            \
-        KOKKOS_LAMBDA(int i, int j, int k)                                          \
-        {                                                                           \
-          i += PV_ixl;                                                              \
-          j += PV_iyl;                                                              \
-          k += PV_izl;                                                              \
-                                                                                    \
-          loop_body;                                                                \
-        };                                                                          \
-                                                                                    \
-      using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
-      MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{PV_diff_x, PV_diff_y, PV_diff_z}});\
-      Kokkos::parallel_for(mdpolicy_3d, lambda_body);                               \
-      Kokkos::fence();                                                              \
-    }                                                                               \
-    i = PV_ixu;                                                                     \
-    j = PV_iyu;                                                                     \
-    k = PV_izu;                                                                     \
-  })                                                                                \
-  (void)i;(void)j;(void)k;                                                          \
-}
-
- /** Loop definition for Kokkos. */
-#define GrGeomOutLoop_kokkos(i, j, k, grgeom, r,                                    \
-  ix, iy, iz, nx, ny, nz, body)                                                     \
-{                                                                                   \
-  if(nx > 0 && ny > 0 && nz > 0)                                                    \
-  {                                                                                 \
-    if(!(GrGeomSolidCellFlagInitialized(grgeom) & (1 << 1)))                        \
-    {                                                                               \
-      CheckCellFlagAllocation(grgeom, nx, ny, nz);                                  \
-      char *outflag = GrGeomSolidCellFlagData(grgeom);                              \
-                                                                                    \
-      GrGeomOctree  *PV_node;                                                       \
-      double PV_ref = pow(2.0, r);                                                  \
-                                                                                    \
-      i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;                                \
-      j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;                                \
-      k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;                                \
-      GrGeomOctreeExteriorNodeLoop(i, j, k, PV_node,                                \
-                                   GrGeomSolidData(grgeom),                         \
-                                   GrGeomSolidOctreeBGLevel(grgeom) + r,            \
-                                   ix, iy, iz, nx, ny, nz,                          \
-                                   TRUE,                                            \
-      {                                                                             \
-        body;                                                                       \
-        outflag[(k - iz) * ny * nx + (j - iy) * nx + (i - ix)] |= (1 << 1);         \
-      });                                                                           \
-      GrGeomSolidCellFlagInitialized(grgeom) |= (1 << 1);                           \
-    }                                                                               \
-    else                                                                            \
-    {                                                                               \
-      char *outflag = GrGeomSolidCellFlagData(grgeom);                              \
-      auto lambda_body =                                                            \
-        KOKKOS_LAMBDA(int i, int j, int k)                                          \
-        {                                                                           \
-          i += ix;                                                                  \
-          j += iy;                                                                  \
-          k += iz;                                                                  \
-                                                                                    \
-          if(outflag[(k - iz) * ny * nx + (j - iy) * nx + (i - ix)] & (1 << 1))     \
-          {                                                                         \
-            body;                                                                   \
-          }                                                                         \
-        };                                                                          \
-                                                                                    \
-      using MDPolicyType_3D = typename Kokkos::Experimental::MDRangePolicy<Kokkos::Experimental::Rank<3> >;\
-      MDPolicyType_3D mdpolicy_3d({{0, 0, 0}}, {{nx, ny, nz}});                     \
-      Kokkos::parallel_for(mdpolicy_3d, lambda_body);                               \
-      Kokkos::fence();                                                              \
-    }                                                                               \
-  }                                                                                 \
-  (void)i;(void)j;(void)k;                                                          \
-}
-
+#define CheckCellFlagAllocation(grgeom, nx, ny, nz)                                            \
+        {                                                                                      \
+          int flagdata_size = sizeof(char) * (nz * ny * nx);                                   \
+          if (GrGeomSolidCellFlagDataSize(grgeom) < flagdata_size)                             \
+          {                                                                                    \
+            char *flagdata = (char*)_ctalloc_device(flagdata_size);                            \
+                                                                                               \
+            if (GrGeomSolidCellFlagDataSize(grgeom) > 0)                                       \
+            memcpy(flagdata, GrGeomSolidCellFlagData(grgeom),                                  \
+                   GrGeomSolidCellFlagDataSize(grgeom));                                       \
+                                                                                               \
+            _tfree_device(GrGeomSolidCellFlagData(grgeom));                                    \
+            GrGeomSolidCellFlagData(grgeom) = flagdata;                                        \
+            GrGeomSolidCellFlagDataSize(grgeom) = flagdata_size;                               \
+          }                                                                                    \
+        }
+
+/** Loop definition for Kokkos. */
+#define BoxLoopI1_kokkos(i, j, k,                                                                           \
+                         ix, iy, iz, nx, ny, nz,                                                            \
+                         i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                  \
+                         loop_body)                                                                         \
+        {                                                                                                   \
+          if (nx > 0 && ny > 0 && nz > 0)                                                                   \
+          {                                                                                                 \
+            DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);                     \
+                                                                                                            \
+            const auto &ref_i1 = i1;                                                                        \
+                                                                                                            \
+            auto lambda_body =                                                                              \
+              KOKKOS_LAMBDA(int i, int j, int k)                                                            \
+            {                                                                                               \
+              const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                                       \
+                             + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                                   \
+                                                                                                            \
+              i += ix;                                                                                      \
+              j += iy;                                                                                      \
+              k += iz;                                                                                      \
+                                                                                                            \
+              loop_body;                                                                                    \
+            };                                                                                              \
+                                                                                                            \
+            using MDPolicyType_3D = typename Kokkos::MDRangePolicy < Kokkos::Rank < 3 > >;                  \
+            MDPolicyType_3D mdpolicy_3d({ { 0, 0, 0 } }, { { nx, ny, nz } });                               \
+            Kokkos::parallel_for(mdpolicy_3d, lambda_body);                                                 \
+                                                                                                            \
+            typedef function_traits < decltype(lambda_body) > traits;                                       \
+            if (!std::is_same < traits::result_type, struct SkipParallelSync > ::value)                     \
+            Kokkos::fence();                                                                                \
+          }                                                                                                 \
+          (void)i; (void)j; (void)k;                                                                        \
+        }
+
+/** Loop definition for Kokkos. */
+#define BoxLoopI2_kokkos(i, j, k,                                                                           \
+                         ix, iy, iz, nx, ny, nz,                                                            \
+                         i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                  \
+                         i2, nx2, ny2, nz2, sx2, sy2, sz2,                                                  \
+                         loop_body)                                                                         \
+        {                                                                                                   \
+          if (nx > 0 && ny > 0 && nz > 0)                                                                   \
+          {                                                                                                 \
+            DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);                     \
+            DeclareInc(PV_jinc_2, PV_kinc_2, nx, ny, nz, nx2, ny2, nz2, sx2, sy2, sz2);                     \
+                                                                                                            \
+            const auto &ref_i1 = i1;                                                                        \
+            const auto &ref_i2 = i2;                                                                        \
+                                                                                                            \
+            auto lambda_body =                                                                              \
+              KOKKOS_LAMBDA(int i, int j, int k)                                                            \
+            {                                                                                               \
+              const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                                       \
+                             + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                                   \
+              const int i2 = k * PV_kinc_2 + (k * ny + j) * PV_jinc_2                                       \
+                             + (k * ny * nx + j * nx + i) * sx2 + ref_i2;                                   \
+                                                                                                            \
+              i += ix;                                                                                      \
+              j += iy;                                                                                      \
+              k += iz;                                                                                      \
+                                                                                                            \
+              loop_body;                                                                                    \
+            };                                                                                              \
+                                                                                                            \
+            using MDPolicyType_3D = typename Kokkos::MDRangePolicy < Kokkos::Rank < 3 > >;                  \
+            MDPolicyType_3D mdpolicy_3d({ { 0, 0, 0 } }, { { nx, ny, nz } });                               \
+            Kokkos::parallel_for(mdpolicy_3d, lambda_body);                                                 \
+                                                                                                            \
+            typedef function_traits < decltype(lambda_body) > traits;                                       \
+            if (!std::is_same < traits::result_type, struct SkipParallelSync > ::value)                     \
+            Kokkos::fence();                                                                                \
+          }                                                                                                 \
+          (void)i; (void)j; (void)k;                                                                        \
+        }
+
+/** Loop definition for Kokkos. */
+#define BoxLoopI3_kokkos(i, j, k,                                                                           \
+                         ix, iy, iz, nx, ny, nz,                                                            \
+                         i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                  \
+                         i2, nx2, ny2, nz2, sx2, sy2, sz2,                                                  \
+                         i3, nx3, ny3, nz3, sx3, sy3, sz3,                                                  \
+                         loop_body)                                                                         \
+        {                                                                                                   \
+          if (nx > 0 && ny > 0 && nz > 0)                                                                   \
+          {                                                                                                 \
+            DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);                     \
+            DeclareInc(PV_jinc_2, PV_kinc_2, nx, ny, nz, nx2, ny2, nz2, sx2, sy2, sz2);                     \
+            DeclareInc(PV_jinc_3, PV_kinc_3, nx, ny, nz, nx3, ny3, nz3, sx3, sy3, sz3);                     \
+                                                                                                            \
+            const auto &ref_i1 = i1;                                                                        \
+            const auto &ref_i2 = i2;                                                                        \
+            const auto &ref_i3 = i3;                                                                        \
+                                                                                                            \
+            auto lambda_body =                                                                              \
+              KOKKOS_LAMBDA(int i, int j, int k)                                                            \
+            {                                                                                               \
+              const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                                       \
+                             + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                                   \
+              const int i2 = k * PV_kinc_2 + (k * ny + j) * PV_jinc_2                                       \
+                             + (k * ny * nx + j * nx + i) * sx2 + ref_i2;                                   \
+              const int i3 = k * PV_kinc_3 + (k * ny + j) * PV_jinc_3                                       \
+                             + (k * ny * nx + j * nx + i) * sx3 + ref_i3;                                   \
+                                                                                                            \
+              i += ix;                                                                                      \
+              j += iy;                                                                                      \
+              k += iz;                                                                                      \
+                                                                                                            \
+              loop_body;                                                                                    \
+            };                                                                                              \
+                                                                                                            \
+            using MDPolicyType_3D = typename Kokkos::MDRangePolicy < Kokkos::Rank < 3 > >;                  \
+            MDPolicyType_3D mdpolicy_3d({ { 0, 0, 0 } }, { { nx, ny, nz } });                               \
+            Kokkos::parallel_for(mdpolicy_3d, lambda_body);                                                 \
+                                                                                                            \
+            typedef function_traits < decltype(lambda_body) > traits;                                       \
+            if (!std::is_same < traits::result_type, struct SkipParallelSync > ::value)                     \
+            Kokkos::fence();                                                                                \
+          }                                                                                                 \
+          (void)i; (void)j; (void)k;                                                                        \
+        }
+
+/** Loop definition for Kokkos. */
+#define BoxLoopReduceI1_kokkos(rslt, i, j, k,                                                                     \
+                               ix, iy, iz, nx, ny, nz,                                                            \
+                               i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                  \
+                               loop_body)                                                                         \
+        {                                                                                                         \
+          if (nx > 0 && ny > 0 && nz > 0)                                                                         \
+          {                                                                                                       \
+            DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);                           \
+                                                                                                                  \
+            const auto &ref_rslt = rslt;                                                                          \
+            const auto &ref_i1 = i1;                                                                              \
+                                                                                                                  \
+            auto lambda_body =                                                                                    \
+              KOKKOS_LAMBDA(int i, int j, int k, double& lsum)                                                    \
+            {                                                                                                     \
+              const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                                             \
+                             + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                                         \
+                                                                                                                  \
+              i += ix;                                                                                            \
+              j += iy;                                                                                            \
+              k += iz;                                                                                            \
+                                                                                                                  \
+              loop_body;                                                                                          \
+            };                                                                                                    \
+                                                                                                                  \
+            auto lambda_outer =                                                                                   \
+              KOKKOS_LAMBDA(int i, int j, int k, double& lsum)                                                    \
+            {                                                                                                     \
+              lambda_body(i, j, k, lsum);                                                                         \
+            };                                                                                                    \
+                                                                                                                  \
+            using MDPolicyType_3D = typename Kokkos::MDRangePolicy < Kokkos::Rank < 3 > >;                        \
+            MDPolicyType_3D mdpolicy_3d({ { 0, 0, 0 } }, { { nx, ny, nz } });                                     \
+            typedef function_traits < decltype(lambda_body) > traits;                                             \
+            if (std::is_same < traits::result_type, struct ReduceSumType < double > > ::value)                    \
+            {                                                                                                     \
+              Kokkos::parallel_reduce(mdpolicy_3d, lambda_outer, rslt);                                           \
+            }                                                                                                     \
+            else if (std::is_same < traits::result_type, struct ReduceMaxType < double > > ::value)               \
+            {                                                                                                     \
+              Kokkos::parallel_reduce(mdpolicy_3d, lambda_outer, Kokkos::Max < double > (rslt));                  \
+            }                                                                                                     \
+            else if (std::is_same < traits::result_type, struct ReduceMinType < double > > ::value)               \
+            {                                                                                                     \
+              Kokkos::parallel_reduce(mdpolicy_3d, lambda_outer, Kokkos::Min < double > (rslt));                  \
+            }                                                                                                     \
+            else                                                                                                  \
+            {                                                                                                     \
+              printf("ERROR at %s:%d: Invalid reduction identifier,                         \
+      likely a problem with a BoxLoopReduce body.", __FILE__, __LINE__);                                          \
+            }                                                                                                     \
+            Kokkos::fence();                                                                                      \
+          }                                                                                                       \
+          (void)i; (void)j; (void)k;                                                                              \
+        }
+
+/** Loop definition for Kokkos. */
+#define BoxLoopReduceI2_kokkos(rslt, i, j, k,                                                                     \
+                               ix, iy, iz, nx, ny, nz,                                                            \
+                               i1, nx1, ny1, nz1, sx1, sy1, sz1,                                                  \
+                               i2, nx2, ny2, nz2, sx2, sy2, sz2,                                                  \
+                               loop_body)                                                                         \
+        {                                                                                                         \
+          if (nx > 0 && ny > 0 && nz > 0)                                                                         \
+          {                                                                                                       \
+            DeclareInc(PV_jinc_1, PV_kinc_1, nx, ny, nz, nx1, ny1, nz1, sx1, sy1, sz1);                           \
+            DeclareInc(PV_jinc_2, PV_kinc_2, nx, ny, nz, nx2, ny2, nz2, sx2, sy2, sz2);                           \
+                                                                                                                  \
+            const auto &ref_rslt = rslt;                                                                          \
+            const auto &ref_i1 = i1;                                                                              \
+            const auto &ref_i2 = i2;                                                                              \
+                                                                                                                  \
+            auto lambda_body =                                                                                    \
+              KOKKOS_LAMBDA(int i, int j, int k, double& lsum)                                                    \
+            {                                                                                                     \
+              const int i1 = k * PV_kinc_1 + (k * ny + j) * PV_jinc_1                                             \
+                             + (k * ny * nx + j * nx + i) * sx1 + ref_i1;                                         \
+              const int i2 = k * PV_kinc_2 + (k * ny + j) * PV_jinc_2                                             \
+                             + (k * ny * nx + j * nx + i) * sx2 + ref_i2;                                         \
+                                                                                                                  \
+              i += ix;                                                                                            \
+              j += iy;                                                                                            \
+              k += iz;                                                                                            \
+                                                                                                                  \
+              loop_body;                                                                                          \
+            };                                                                                                    \
+                                                                                                                  \
+            auto lambda_outer =                                                                                   \
+              KOKKOS_LAMBDA(int i, int j, int k, double& lsum)                                                    \
+            {                                                                                                     \
+              lambda_body(i, j, k, lsum);                                                                         \
+            };                                                                                                    \
+                                                                                                                  \
+            using MDPolicyType_3D = typename Kokkos::MDRangePolicy < Kokkos::Rank < 3 > >;                        \
+            MDPolicyType_3D mdpolicy_3d({ { 0, 0, 0 } }, { { nx, ny, nz } });                                     \
+            typedef function_traits < decltype(lambda_body) > traits;                                             \
+            if (std::is_same < traits::result_type, struct ReduceSumType < double > > ::value)                    \
+            {                                                                                                     \
+              Kokkos::parallel_reduce(mdpolicy_3d, lambda_outer, rslt);                                           \
+            }                                                                                                     \
+            else if (std::is_same < traits::result_type, struct ReduceMaxType < double > > ::value)               \
+            {                                                                                                     \
+              Kokkos::parallel_reduce(mdpolicy_3d, lambda_outer, Kokkos::Max < double > (rslt));                  \
+            }                                                                                                     \
+            else if (std::is_same < traits::result_type, struct ReduceMinType < double > > ::value)               \
+            {                                                                                                     \
+              Kokkos::parallel_reduce(mdpolicy_3d, lambda_outer, Kokkos::Min < double > (rslt));                  \
+            }                                                                                                     \
+            else                                                                                                  \
+            {                                                                                                     \
+              printf("ERROR at %s:%d: Invalid reduction identifier,                         \
+      likely a problem with a BoxLoopReduce body.", __FILE__, __LINE__);                                          \
+            }                                                                                                     \
+            Kokkos::fence();                                                                                      \
+          }                                                                                                       \
+          (void)i; (void)j; (void)k;                                                                              \
+        }
+
+/** Loop definition for Kokkos. */
+#define GrGeomInLoopBoxes_kokkos(i, j, k,                                                                           \
+                                 grgeom, ix, iy, iz, nx, ny, nz, loop_body)                                         \
+        {                                                                                                           \
+          BoxArray* boxes = GrGeomSolidInteriorBoxes(grgeom);                                                       \
+          int ix_bxs = BoxArrayMinCell(boxes, 0);                                                                   \
+          int iy_bxs = BoxArrayMinCell(boxes, 1);                                                                   \
+          int iz_bxs = BoxArrayMinCell(boxes, 2);                                                                   \
+                                                                                                                    \
+          int nx_bxs = BoxArrayMaxCell(boxes, 0) - ix_bxs + 1;                                                      \
+          int ny_bxs = BoxArrayMaxCell(boxes, 1) - iy_bxs + 1;                                                      \
+          int nz_bxs = BoxArrayMaxCell(boxes, 2) - iz_bxs + 1;                                                      \
+                                                                                                                    \
+          if (!(GrGeomSolidCellFlagInitialized(grgeom) & 1))                                                        \
+          {                                                                                                         \
+            CheckCellFlagAllocation(grgeom, nx_bxs, ny_bxs, nz_bxs);                                                \
+            char *inflag = GrGeomSolidCellFlagData(grgeom);                                                         \
+                                                                                                                    \
+            for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                                            \
+            {                                                                                                       \
+              Box box = BoxArrayGetBox(boxes, PV_box);                                                              \
+              int PV_ixl = box.lo[0];                                                                               \
+              int PV_iyl = box.lo[1];                                                                               \
+              int PV_izl = box.lo[2];                                                                               \
+              int PV_ixu = box.up[0];                                                                               \
+              int PV_iyu = box.up[1];                                                                               \
+              int PV_izu = box.up[2];                                                                               \
+                                                                                                                    \
+              if (PV_ixl <= PV_ixu && PV_iyl <= PV_iyu && PV_izl <= PV_izu)                                         \
+              {                                                                                                     \
+                int PV_nx = PV_ixu - PV_ixl + 1;                                                                    \
+                int PV_ny = PV_iyu - PV_iyl + 1;                                                                    \
+                int PV_nz = PV_izu - PV_izl + 1;                                                                    \
+                                                                                                                    \
+                Globals *globals = ::globals;                                                                       \
+                auto lambda_body =                                                                                  \
+                  KOKKOS_LAMBDA(int i, int j, int k)                                                                \
+                {                                                                                                   \
+                  i += PV_ixl;                                                                                      \
+                  j += PV_iyl;                                                                                      \
+                  k += PV_izl;                                                                                      \
+                                                                                                                    \
+                  /* Set inflag for all cells in boxes regardless of loop limits */                                 \
+                  inflag[(k - iz_bxs) * ny_bxs * nx_bxs +                                                           \
+                         (j - iy_bxs) * nx_bxs + (i - ix_bxs)] |= 1;                                                \
+                                                                                                                    \
+                  /* Only evaluate loop body if the cell is within loop limits */                                   \
+                  if (i >= ix && j >= iy && k >= iz &&                                                              \
+                      i < ix + nx && j < iy + ny && k < iz + nz)                                                    \
+                  {                                                                                                 \
+                    loop_body;                                                                                      \
+                  }                                                                                                 \
+                };                                                                                                  \
+                                                                                                                    \
+                using MDPolicyType_3D = typename Kokkos::MDRangePolicy < Kokkos::Rank < 3 > >;                      \
+                MDPolicyType_3D mdpolicy_3d({ { 0, 0, 0 } }, { { PV_nx, PV_ny, PV_nz } });                          \
+                Kokkos::parallel_for(mdpolicy_3d, lambda_body);                                                     \
+                Kokkos::fence();                                                                                    \
+              }                                                                                                     \
+            }                                                                                                       \
+            GrGeomSolidCellFlagInitialized(grgeom) |= 1;                                                            \
+          }                                                                                                         \
+          else                                                                                                      \
+          {                                                                                                         \
+            int ixl_gpu = pfmax(ix, ix_bxs);                                                                        \
+            int iyl_gpu = pfmax(iy, iy_bxs);                                                                        \
+            int izl_gpu = pfmax(iz, iz_bxs);                                                                        \
+            int nx_gpu = pfmin((ix + nx - 1), BoxArrayMaxCell(boxes, 0)) - ixl_gpu + 1;                             \
+            int ny_gpu = pfmin((iy + ny - 1), BoxArrayMaxCell(boxes, 1)) - iyl_gpu + 1;                             \
+            int nz_gpu = pfmin((iz + nz - 1), BoxArrayMaxCell(boxes, 2)) - izl_gpu + 1;                             \
+                                                                                                                    \
+            if (nx_gpu > 0 && ny_gpu > 0 && nz_gpu > 0)                                                             \
+            {                                                                                                       \
+              Globals *globals = ::globals;                                                                         \
+              char *inflag = GrGeomSolidCellFlagData(grgeom);                                                       \
+              auto lambda_body =                                                                                    \
+                KOKKOS_LAMBDA(int i, int j, int k)                                                                  \
+              {                                                                                                     \
+                i += ixl_gpu;                                                                                       \
+                j += iyl_gpu;                                                                                       \
+                k += izl_gpu;                                                                                       \
+                if (inflag[(k - iz_bxs) * ny_bxs * nx_bxs +                                                         \
+                           (j - iy_bxs) * nx_bxs + (i - ix_bxs)] & 1)                                               \
+                {                                                                                                   \
+                  loop_body;                                                                                        \
+                }                                                                                                   \
+              };                                                                                                    \
+                                                                                                                    \
+              using MDPolicyType_3D = typename Kokkos::MDRangePolicy < Kokkos::Rank < 3 > >;                        \
+              MDPolicyType_3D mdpolicy_3d({ { 0, 0, 0 } }, { { nx_gpu, ny_gpu, nz_gpu } });                         \
+              Kokkos::parallel_for(mdpolicy_3d, lambda_body);                                                       \
+              Kokkos::fence();                                                                                      \
+            }                                                                                                       \
+          }                                                                                                         \
+          (void)i; (void)j; (void)k;                                                                                \
+        }
+
+/** Loop definition for Kokkos. */
+#define GrGeomSurfLoopBoxes_kokkos(i, j, k, fdir, grgeom,                                                             \
+                                   ix, iy, iz, nx, ny, nz, loop_body)                                                 \
+        {                                                                                                             \
+          for (int PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                                                     \
+          {                                                                                                           \
+            const int *fdir = FDIR_TABLE[PV_f];                                                                       \
+                                                                                                                      \
+            BoxArray* boxes = GrGeomSolidSurfaceBoxes(grgeom, PV_f);                                                  \
+            int ix_bxs = BoxArrayMinCell(boxes, 0);                                                                   \
+            int iy_bxs = BoxArrayMinCell(boxes, 1);                                                                   \
+            int iz_bxs = BoxArrayMinCell(boxes, 2);                                                                   \
+                                                                                                                      \
+            int nx_bxs = BoxArrayMaxCell(boxes, 0) - ix_bxs + 1;                                                      \
+            int ny_bxs = BoxArrayMaxCell(boxes, 1) - iy_bxs + 1;                                                      \
+            int nz_bxs = BoxArrayMaxCell(boxes, 2) - iz_bxs + 1;                                                      \
+                                                                                                                      \
+            if (!(GrGeomSolidCellFlagInitialized(grgeom) & (1 << (2 + PV_f))))                                        \
+            {                                                                                                         \
+              CheckCellFlagAllocation(grgeom, nx_bxs, ny_bxs, nz_bxs);                                                \
+              char *surfflag = GrGeomSolidCellFlagData(grgeom);                                                       \
+                                                                                                                      \
+              for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                                            \
+              {                                                                                                       \
+                Box box = BoxArrayGetBox(boxes, PV_box);                                                              \
+                int PV_ixl = box.lo[0];                                                                               \
+                int PV_iyl = box.lo[1];                                                                               \
+                int PV_izl = box.lo[2];                                                                               \
+                int PV_ixu = box.up[0];                                                                               \
+                int PV_iyu = box.up[1];                                                                               \
+                int PV_izu = box.up[2];                                                                               \
+                                                                                                                      \
+                if (PV_ixl <= PV_ixu && PV_iyl <= PV_iyu && PV_izl <= PV_izu)                                         \
+                {                                                                                                     \
+                  int PV_nx = PV_ixu - PV_ixl + 1;                                                                    \
+                  int PV_ny = PV_iyu - PV_iyl + 1;                                                                    \
+                  int PV_nz = PV_izu - PV_izl + 1;                                                                    \
+                                                                                                                      \
+                  const int _fdir0 = fdir[0];                                                                         \
+                  const int _fdir1 = fdir[1];                                                                         \
+                  const int _fdir2 = fdir[2];                                                                         \
+                                                                                                                      \
+                  auto lambda_body =                                                                                  \
+                    KOKKOS_LAMBDA(int i, int j, int k)                                                                \
+                  {                                                                                                   \
+                    i += PV_ixl;                                                                                      \
+                    j += PV_iyl;                                                                                      \
+                    k += PV_izl;                                                                                      \
+                                                                                                                      \
+                    /* Set surfflag for all cells in boxes regardless of loop limits */                               \
+                    surfflag[(k - iz_bxs) * ny_bxs * nx_bxs +                                                         \
+                             (j - iy_bxs) * nx_bxs + (i - ix_bxs)] |= (1 << (2 + PV_f));                              \
+                                                                                                                      \
+                    /* Only evaluate loop body if the cell is within loop limits */                                   \
+                    if (i >= ix && j >= iy && k >= iz &&                                                              \
+                        i < ix + nx && j < iy + ny && k < iz + nz)                                                    \
+                    {                                                                                                 \
+                      const int fdir[3] = { _fdir0, _fdir1, _fdir2 };                                                 \
+                      loop_body;                                                                                      \
+                      (void)fdir;                                                                                     \
+                    }                                                                                                 \
+                  };                                                                                                  \
+                                                                                                                      \
+                  using MDPolicyType_3D = typename Kokkos::MDRangePolicy < Kokkos::Rank < 3 > >;                      \
+                  MDPolicyType_3D mdpolicy_3d({ { 0, 0, 0 } }, { { PV_nx, PV_ny, PV_nz } });                          \
+                  Kokkos::parallel_for(mdpolicy_3d, lambda_body);                                                     \
+                  Kokkos::fence();                                                                                    \
+                }                                                                                                     \
+              }                                                                                                       \
+              GrGeomSolidCellFlagInitialized(grgeom) |= (1 << (2 + PV_f));                                            \
+            }                                                                                                         \
+            else                                                                                                      \
+            {                                                                                                         \
+              int ixl_gpu = pfmax(ix, ix_bxs);                                                                        \
+              int iyl_gpu = pfmax(iy, iy_bxs);                                                                        \
+              int izl_gpu = pfmax(iz, iz_bxs);                                                                        \
+              int nx_gpu = pfmin((ix + nx - 1), BoxArrayMaxCell(boxes, 0)) - ixl_gpu + 1;                             \
+              int ny_gpu = pfmin((iy + ny - 1), BoxArrayMaxCell(boxes, 1)) - iyl_gpu + 1;                             \
+              int nz_gpu = pfmin((iz + nz - 1), BoxArrayMaxCell(boxes, 2)) - izl_gpu + 1;                             \
+                                                                                                                      \
+              if (nx_gpu > 0 && ny_gpu > 0 && nz_gpu > 0)                                                             \
+              {                                                                                                       \
+                char *surfflag = GrGeomSolidCellFlagData(grgeom);                                                     \
+                                                                                                                      \
+                const int _fdir0 = fdir[0];                                                                           \
+                const int _fdir1 = fdir[1];                                                                           \
+                const int _fdir2 = fdir[2];                                                                           \
+                                                                                                                      \
+                auto lambda_body =                                                                                    \
+                  KOKKOS_LAMBDA(int i, int j, int k)                                                                  \
+                {                                                                                                     \
+                  i += ixl_gpu;                                                                                       \
+                  j += iyl_gpu;                                                                                       \
+                  k += izl_gpu;                                                                                       \
+                                                                                                                      \
+                  if (surfflag[(k - iz_bxs) * ny_bxs * nx_bxs +                                                       \
+                               (j - iy_bxs) * nx_bxs + (i - ix_bxs)] & (1 << (2 + PV_f)))                             \
+                  {                                                                                                   \
+                    const int fdir[3] = { _fdir0, _fdir1, _fdir2 };                                                   \
+                    loop_body;                                                                                        \
+                    (void)fdir;                                                                                       \
+                  }                                                                                                   \
+                };                                                                                                    \
+                                                                                                                      \
+                using MDPolicyType_3D = typename Kokkos::MDRangePolicy < Kokkos::Rank < 3 > >;                        \
+                MDPolicyType_3D mdpolicy_3d({ { 0, 0, 0 } }, { { nx_gpu, ny_gpu, nz_gpu } });                         \
+                Kokkos::parallel_for(mdpolicy_3d, lambda_body);                                                       \
+                Kokkos::fence();                                                                                      \
+              }                                                                                                       \
+            }                                                                                                         \
+          }                                                                                                           \
+          (void)i; (void)j; (void)k;                                                                                  \
+        }
+
+/** Loop definition for Kokkos. */
+#define GrGeomPatchLoopBoxes_kokkos(i, j, k, fdir, grgeom, patch_num,                                                  \
+                                    ix, iy, iz, nx, ny, nz, loop_body)                                                 \
+        {                                                                                                              \
+          for (int PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                                                      \
+          {                                                                                                            \
+            const int *fdir = FDIR_TABLE[PV_f];                                                                        \
+                                                                                                                       \
+            int n_prev = 0;                                                                                            \
+            BoxArray* boxes = GrGeomSolidPatchBoxes(grgeom, patch_num, PV_f);                                          \
+            for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                                               \
+            {                                                                                                          \
+              Box box = BoxArrayGetBox(boxes, PV_box);                                                                 \
+              /* find octree and region intersection */                                                                \
+              int PV_ixl = pfmax(ix, box.lo[0]);                                                                       \
+              int PV_iyl = pfmax(iy, box.lo[1]);                                                                       \
+              int PV_izl = pfmax(iz, box.lo[2]);                                                                       \
+              int PV_ixu = pfmin((ix + nx - 1), box.up[0]);                                                            \
+              int PV_iyu = pfmin((iy + ny - 1), box.up[1]);                                                            \
+              int PV_izu = pfmin((iz + nz - 1), box.up[2]);                                                            \
+                                                                                                                       \
+              if (PV_ixl <= PV_ixu && PV_iyl <= PV_iyu && PV_izl <= PV_izu)                                            \
+              {                                                                                                        \
+                int PV_diff_x = PV_ixu - PV_ixl;                                                                       \
+                int PV_diff_y = PV_iyu - PV_iyl;                                                                       \
+                int PV_diff_z = PV_izu - PV_izl;                                                                       \
+                                                                                                                       \
+                int nx = PV_diff_x + 1;                                                                                \
+                int ny = PV_diff_y + 1;                                                                                \
+                int nz = PV_diff_z + 1;                                                                                \
+                                                                                                                       \
+                const int _fdir0 = fdir[0];                                                                            \
+                const int _fdir1 = fdir[1];                                                                            \
+                const int _fdir2 = fdir[2];                                                                            \
+                                                                                                                       \
+                auto lambda_body =                                                                                     \
+                  KOKKOS_LAMBDA(int i, int j, int k)                                                                   \
+                {                                                                                                      \
+                  const int fdir[3] = { _fdir0, _fdir1, _fdir2 };                                                      \
+                  int ival = n_prev + k * ny * nx + j * nx + i;                                                        \
+                                                                                                                       \
+                  i += PV_ixl;                                                                                         \
+                  j += PV_iyl;                                                                                         \
+                  k += PV_izl;                                                                                         \
+                                                                                                                       \
+                  loop_body;                                                                                           \
+                  (void)fdir;                                                                                          \
+                };                                                                                                     \
+                n_prev += nz * ny * nx;                                                                                \
+                                                                                                                       \
+                using MDPolicyType_3D = typename Kokkos::MDRangePolicy < Kokkos::Rank < 3 > >;                         \
+                MDPolicyType_3D mdpolicy_3d({ { 0, 0, 0 } }, { { nx, ny, nz } });                                      \
+                Kokkos::parallel_for(mdpolicy_3d, lambda_body);                                                        \
+                Kokkos::fence();                                                                                       \
+              }                                                                                                        \
+            }                                                                                                          \
+          }                                                                                                            \
+          (void)i; (void)j; (void)k;                                                                                   \
+        }
+
+/** Loop definition for Kokkos. */
+#define GrGeomPatchLoopBoxesNoFdir_kokkos(i, j, k, grgeom, patch_num, ovrlnd,                                                \
+                                          ix, iy, iz, nx, ny, nz, locals, setup,                                             \
+                                          f_left, f_right, f_down, f_up, f_back, f_front, finalize)                          \
+        {                                                                                                                    \
+          int n_ival = 0;                                                                                                    \
+          for (int PV_f = 0; PV_f < GrGeomOctreeNumFaces; PV_f++)                                                            \
+          {                                                                                                                  \
+            BoxArray* boxes = GrGeomSolidPatchBoxes(grgeom, patch_num, PV_f);                                                \
+                                                                                                                             \
+            int ix_bxs = BoxArrayMinCell(boxes, 0);                                                                          \
+            int iy_bxs = BoxArrayMinCell(boxes, 1);                                                                          \
+            int iz_bxs = BoxArrayMinCell(boxes, 2);                                                                          \
+                                                                                                                             \
+            int nx_bxs = BoxArrayMaxCell(boxes, 0) - ix_bxs + 1;                                                             \
+            int ny_bxs = BoxArrayMaxCell(boxes, 1) - iy_bxs + 1;                                                             \
+            int nz_bxs = BoxArrayMaxCell(boxes, 2) - iz_bxs + 1;                                                             \
+                                                                                                                             \
+            int patch_loc;                                                                                                   \
+            if (ovrlnd)                                                                                                      \
+            patch_loc = GrGeomSolidNumPatches(grgeom) +patch_num;                                                            \
+            else                                                                                                             \
+            patch_loc = patch_num;                                                                                           \
+                                                                                                                             \
+            int *ptr_ival = GrGeomSolidCellIval(grgeom, patch_loc, PV_f);                                                    \
+            if (!(ptr_ival))                                                                                                 \
+            {                                                                                                                \
+              GrGeomSolidCellIval(grgeom, patch_loc, PV_f) =                                                                 \
+                (int*)_talloc_device(sizeof(int) * nx_bxs * ny_bxs * nz_bxs);                                                \
+                                                                                                                             \
+              ptr_ival = GrGeomSolidCellIval(grgeom, patch_loc, PV_f);                                                       \
+              for (int idx = 0; idx < nx_bxs * ny_bxs * nz_bxs; idx++)                                                       \
+              ptr_ival[idx] = -1;                                                                                            \
+                                                                                                                             \
+              for (int PV_box = 0; PV_box < BoxArraySize(boxes); PV_box++)                                                   \
+              {                                                                                                              \
+                Box box = BoxArrayGetBox(boxes, PV_box);                                                                     \
+                int PV_ixl = pfmax(ix, box.lo[0]);                                                                           \
+                int PV_iyl = pfmax(iy, box.lo[1]);                                                                           \
+                int PV_izl = pfmax(iz, box.lo[2]);                                                                           \
+                int PV_ixu = pfmin((ix + nx - 1), box.up[0]);                                                                \
+                int PV_iyu = pfmin((iy + ny - 1), box.up[1]);                                                                \
+                int PV_izu = pfmin((iz + nz - 1), box.up[2]);                                                                \
+                                                                                                                             \
+                for (k = PV_izl; k <= PV_izu; k++)                                                                           \
+                for (j = PV_iyl; j <= PV_iyu; j++)                                                                           \
+                for (i = PV_ixl; i <= PV_ixu; i++)                                                                           \
+                {                                                                                                            \
+                  UNPACK(locals);                                                                                            \
+                  setup;                                                                                                     \
+                  switch (PV_f)                                                                                              \
+                  {                                                                                                          \
+                  f_left;                                                                                                    \
+                  f_right;                                                                                                   \
+                  f_down;                                                                                                    \
+                  f_up;                                                                                                      \
+                  f_back;                                                                                                    \
+                  f_front;                                                                                                   \
+                  }                                                                                                          \
+                  finalize;                                                                                                  \
+                  ptr_ival[(k - iz_bxs) * ny_bxs * nx_bxs + (j - iy_bxs) *                                                   \
+                           nx_bxs + (i - ix_bxs)] = n_ival++;                                                                \
+                }                                                                                                            \
+              }                                                                                                              \
+            }                                                                                                                \
+            else                                                                                                             \
+            {                                                                                                                \
+              int ixl_gpu = pfmax(ix, ix_bxs);                                                                               \
+              int iyl_gpu = pfmax(iy, iy_bxs);                                                                               \
+              int izl_gpu = pfmax(iz, iz_bxs);                                                                               \
+              int nx_gpu = pfmin((ix + nx - 1), BoxArrayMaxCell(boxes, 0)) - ixl_gpu + 1;                                    \
+              int ny_gpu = pfmin((iy + ny - 1), BoxArrayMaxCell(boxes, 1)) - iyl_gpu + 1;                                    \
+              int nz_gpu = pfmin((iz + nz - 1), BoxArrayMaxCell(boxes, 2)) - izl_gpu + 1;                                    \
+                                                                                                                             \
+              if (nx_gpu > 0 && ny_gpu > 0 && nz_gpu > 0)                                                                    \
+              {                                                                                                              \
+                auto lambda_body =                                                                                           \
+                  KOKKOS_LAMBDA(int i, int j, int k)                                                                         \
+                {                                                                                                            \
+                  i += ixl_gpu;                                                                                              \
+                  j += iyl_gpu;                                                                                              \
+                  k += izl_gpu;                                                                                              \
+                                                                                                                             \
+                  int ival = ptr_ival[(k - iz_bxs) * ny_bxs * nx_bxs +                                                       \
+                                      (j - iy_bxs) * nx_bxs + (i - ix_bxs)];                                                 \
+                  if (ival >= 0)                                                                                             \
+                  {                                                                                                          \
+                    UNPACK(locals);                                                                                          \
+                    setup;                                                                                                   \
+                    switch (PV_f)                                                                                            \
+                    {                                                                                                        \
+                    f_left;                                                                                                  \
+                    f_right;                                                                                                 \
+                    f_down;                                                                                                  \
+                    f_up;                                                                                                    \
+                    f_back;                                                                                                  \
+                    f_front;                                                                                                 \
+                    }                                                                                                        \
+                    finalize;                                                                                                \
+                  }                                                                                                          \
+                };                                                                                                           \
+                                                                                                                             \
+                using MDPolicyType_3D = typename Kokkos::MDRangePolicy < Kokkos::Rank < 3 > >;                               \
+                MDPolicyType_3D mdpolicy_3d({ { 0, 0, 0 } }, { { nx_gpu, ny_gpu, nz_gpu } });                                \
+                Kokkos::parallel_for(mdpolicy_3d, lambda_body);                                                              \
+                Kokkos::fence();                                                                                             \
+              }                                                                                                              \
+            }                                                                                                                \
+          }                                                                                                                  \
+          (void)i; (void)j; (void)k;                                                                                         \
+        }
+
+/** Loop definition for Kokkos. */
+#define GrGeomOctreeExteriorNodeLoop_kokkos(i, j, k, node, octree, level,                                                      \
+                                            ix, iy, iz, nx, ny, nz, val_test, loop_body)                                       \
+        {                                                                                                                      \
+          int PV_i, PV_j, PV_k, PV_l;                                                                                          \
+          int PV_ixl, PV_iyl, PV_izl, PV_ixu, PV_iyu, PV_izu;                                                                  \
+                                                                                                                               \
+          PV_i = i;                                                                                                            \
+          PV_j = j;                                                                                                            \
+          PV_k = k;                                                                                                            \
+                                                                                                                               \
+          GrGeomOctreeExteriorLoop(PV_i, PV_j, PV_k, PV_l, node, octree, level, val_test,                                      \
+    {                                                                                                                          \
+      if ((PV_i >= ix) && (PV_i < (ix + nx)) &&                                                                                \
+          (PV_j >= iy) && (PV_j < (iy + ny)) &&                                                                                \
+          (PV_k >= iz) && (PV_k < (iz + nz)))                                                                                  \
+      {                                                                                                                        \
+        i = PV_i;                                                                                                              \
+        j = PV_j;                                                                                                              \
+        k = PV_k;                                                                                                              \
+        loop_body;                                                                                                             \
+      }                                                                                                                        \
+    },                                                                                                                         \
+    {                                                                                                                          \
+      /* find octree and region intersection */                                                                                \
+      PV_ixl = pfmax(ix, PV_i);                                                                                                \
+      PV_iyl = pfmax(iy, PV_j);                                                                                                \
+      PV_izl = pfmax(iz, PV_k);                                                                                                \
+      PV_ixu = pfmin((ix + nx), (PV_i + (int)PV_inc));                                                                         \
+      PV_iyu = pfmin((iy + ny), (PV_j + (int)PV_inc));                                                                         \
+      PV_izu = pfmin((iz + nz), (PV_k + (int)PV_inc));                                                                         \
+                                                                                                                               \
+      if (PV_ixl < PV_ixu && PV_iyl < PV_iyu && PV_izl < PV_izu)                                                               \
+      {                                                                                                                        \
+        const int PV_diff_x = PV_ixu - PV_ixl;                                                                                 \
+        const int PV_diff_y = PV_iyu - PV_iyl;                                                                                 \
+        const int PV_diff_z = PV_izu - PV_izl;                                                                                 \
+                                                                                                                               \
+        auto lambda_body =                                                                                                     \
+          KOKKOS_LAMBDA(int i, int j, int k)                                                                                   \
+        {                                                                                                                      \
+          i += PV_ixl;                                                                                                         \
+          j += PV_iyl;                                                                                                         \
+          k += PV_izl;                                                                                                         \
+                                                                                                                               \
+          loop_body;                                                                                                           \
+        };                                                                                                                     \
+                                                                                                                               \
+        using MDPolicyType_3D = typename Kokkos::MDRangePolicy < Kokkos::Rank < 3 > >;                                         \
+        MDPolicyType_3D mdpolicy_3d({ { 0, 0, 0 } }, { { PV_diff_x, PV_diff_y, PV_diff_z } });                                 \
+        Kokkos::parallel_for(mdpolicy_3d, lambda_body);                                                                        \
+        Kokkos::fence();                                                                                                       \
+      }                                                                                                                        \
+      i = PV_ixu;                                                                                                              \
+      j = PV_iyu;                                                                                                              \
+      k = PV_izu;                                                                                                              \
+    })                                                                                                                         \
+          (void)i; (void)j; (void)k;                                                                                           \
+        }
+
+/** Loop definition for Kokkos. */
+#define GrGeomOutLoop_kokkos(i, j, k, grgeom, r,                                                                \
+                             ix, iy, iz, nx, ny, nz, body)                                                      \
+        {                                                                                                       \
+          if (nx > 0 && ny > 0 && nz > 0)                                                                       \
+          {                                                                                                     \
+            if (!(GrGeomSolidCellFlagInitialized(grgeom) & (1 << 1)))                                           \
+            {                                                                                                   \
+              CheckCellFlagAllocation(grgeom, nx, ny, nz);                                                      \
+              char *outflag = GrGeomSolidCellFlagData(grgeom);                                                  \
+                                                                                                                \
+              GrGeomOctree  *PV_node;                                                                           \
+              double PV_ref = pow(2.0, r);                                                                      \
+                                                                                                                \
+              i = GrGeomSolidOctreeIX(grgeom) * (int)PV_ref;                                                    \
+              j = GrGeomSolidOctreeIY(grgeom) * (int)PV_ref;                                                    \
+              k = GrGeomSolidOctreeIZ(grgeom) * (int)PV_ref;                                                    \
+              GrGeomOctreeExteriorNodeLoop(i, j, k, PV_node,                                                    \
+                                           GrGeomSolidData(grgeom),                                             \
+                                           GrGeomSolidOctreeBGLevel(grgeom) + r,                                \
+                                           ix, iy, iz, nx, ny, nz,                                              \
+                                           TRUE,                                                                \
+        {                                                                                                       \
+          body;                                                                                                 \
+          outflag[(k - iz) * ny * nx + (j - iy) * nx + (i - ix)] |= (1 << 1);                                   \
+        });                                                                                                     \
+              GrGeomSolidCellFlagInitialized(grgeom) |= (1 << 1);                                               \
+            }                                                                                                   \
+            else                                                                                                \
+            {                                                                                                   \
+              char *outflag = GrGeomSolidCellFlagData(grgeom);                                                  \
+              auto lambda_body =                                                                                \
+                KOKKOS_LAMBDA(int i, int j, int k)                                                              \
+              {                                                                                                 \
+                i += ix;                                                                                        \
+                j += iy;                                                                                        \
+                k += iz;                                                                                        \
+                                                                                                                \
+                if (outflag[(k - iz) * ny * nx + (j - iy) * nx + (i - ix)] & (1 << 1))                          \
+                {                                                                                               \
+                  body;                                                                                         \
+                }                                                                                               \
+              };                                                                                                \
+                                                                                                                \
+              using MDPolicyType_3D = typename Kokkos::MDRangePolicy < Kokkos::Rank < 3 > >;                    \
+              MDPolicyType_3D mdpolicy_3d({ { 0, 0, 0 } }, { { nx, ny, nz } });                                 \
+              Kokkos::parallel_for(mdpolicy_3d, lambda_body);                                                   \
+              Kokkos::fence();                                                                                  \
+            }                                                                                                   \
+          }                                                                                                     \
+          (void)i; (void)j; (void)k;                                                                            \
+        }
 }
 #endif // PF_KOKKOSLOOPS_H
diff --git a/pfsimulator/parflow_lib/pf_kokkosmalloc.h b/pfsimulator/parflow_lib/pf_kokkosmalloc.h
index 71504d7b4..34a826cd3 100644
--- a/pfsimulator/parflow_lib/pf_kokkosmalloc.h
+++ b/pfsimulator/parflow_lib/pf_kokkosmalloc.h
@@ -37,10 +37,10 @@
 #define tfree_amps_kokkos(ptr) amps_TFree_managed(ptr)
 
 #define talloc_kokkos(type, count) \
-  ((count) ? (type*)_talloc_device(sizeof(type) * (unsigned int)(count)) : NULL)
+        ((count) ? (type*)_talloc_device(sizeof(type) * (unsigned int)(count)) : NULL)
 
 #define ctalloc_kokkos(type, count) \
-  ((count) ? (type*)_ctalloc_device(sizeof(type) * (unsigned int)(count)) : NULL)
+        ((count) ? (type*)_ctalloc_device(sizeof(type) * (unsigned int)(count)) : NULL)
 
 #define tfree_kokkos(ptr) if (ptr) _tfree_device(ptr); else {}
 
diff --git a/pfsimulator/parflow_lib/pf_module.c b/pfsimulator/parflow_lib/pf_module.c
index 66c2a1ced..ace84efbb 100644
--- a/pfsimulator/parflow_lib/pf_module.c
+++ b/pfsimulator/parflow_lib/pf_module.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Member functions for the pf_module class.
@@ -52,12 +52,12 @@ PFModule  *NewPFModule(
 
   new_module = talloc(PFModule, 1);
 
-  (new_module->call) = (void (*)())call;
-  (new_module->init_instance_xtra) = (void (*)())init_instance_xtra;
-  (new_module->free_instance_xtra) = (void (*)())free_instance_xtra;
-  (new_module->new_public_xtra) = (void (*)())new_public_xtra;
-  (new_module->free_public_xtra) = (void (*)())free_public_xtra;
-  (new_module->sizeof_temp_data) = (int (*)())sizeof_temp_data;
+  (new_module->call) = (void (*)()) call;
+  (new_module->init_instance_xtra) = (void (*)()) init_instance_xtra;
+  (new_module->free_instance_xtra) = (void (*)()) free_instance_xtra;
+  (new_module->new_public_xtra) = (void (*)()) new_public_xtra;
+  (new_module->free_public_xtra) = (void (*)()) free_public_xtra;
+  (new_module->sizeof_temp_data) = (int (*)()) sizeof_temp_data;
 
   PFModuleInstanceXtra(new_module) = instance_xtra;
   PFModulePublicXtra(new_module) = public_xtra;
@@ -66,29 +66,29 @@ PFModule  *NewPFModule(
 }
 
 PFModule  *NewPFModuleExtended(
-                       void *call,
-                       void *init_instance_xtra,
-                       void *free_instance_xtra,
-                       void *new_public_xtra,
-                       void *free_public_xtra,
-                       void *sizeof_temp_data,
-		       void *output,
-		       void *output_static,
-                       void *instance_xtra,
-                       void *public_xtra)
+                               void *call,
+                               void *init_instance_xtra,
+                               void *free_instance_xtra,
+                               void *new_public_xtra,
+                               void *free_public_xtra,
+                               void *sizeof_temp_data,
+                               void *output,
+                               void *output_static,
+                               void *instance_xtra,
+                               void *public_xtra)
 {
   PFModule         *new_module;
 
   new_module = talloc(PFModule, 1);
 
-  (new_module->call) = (void (*)())call;
-  (new_module->init_instance_xtra) = (void (*)())init_instance_xtra;
-  (new_module->free_instance_xtra) = (void (*)())free_instance_xtra;
-  (new_module->new_public_xtra) = (void (*)())new_public_xtra;
-  (new_module->free_public_xtra) = (void (*)())free_public_xtra;
-  (new_module->sizeof_temp_data) = (int (*)())sizeof_temp_data;
-  (new_module->output) = (void (*)())output;
-  (new_module->output_static) = (void (*)())output_static;
+  (new_module->call) = (void (*)()) call;
+  (new_module->init_instance_xtra) = (void (*)()) init_instance_xtra;
+  (new_module->free_instance_xtra) = (void (*)()) free_instance_xtra;
+  (new_module->new_public_xtra) = (void (*)()) new_public_xtra;
+  (new_module->free_public_xtra) = (void (*)()) free_public_xtra;
+  (new_module->sizeof_temp_data) = (int (*)()) sizeof_temp_data;
+  (new_module->output) = (void (*)()) output;
+  (new_module->output_static) = (void (*)()) output_static;
 
   PFModuleInstanceXtra(new_module) = instance_xtra;
   PFModulePublicXtra(new_module) = public_xtra;
@@ -104,15 +104,15 @@ PFModule  *NewPFModuleExtended(
 PFModule  *DupPFModule(PFModule *pf_module)
 {
   return NewPFModuleExtended((void*)(pf_module->call),
-			     (void*)(pf_module->init_instance_xtra),
-			     (void*)(pf_module->free_instance_xtra),
-			     (void*)(pf_module->new_public_xtra),
-			     (void*)(pf_module->free_public_xtra),
-			     (void*)(pf_module->sizeof_temp_data),
-			     (void*)(pf_module->output),
-			     (void*)(pf_module->output_static),
-			     PFModuleInstanceXtra(pf_module),
-			     PFModulePublicXtra(pf_module));
+                             (void*)(pf_module->init_instance_xtra),
+                             (void*)(pf_module->free_instance_xtra),
+                             (void*)(pf_module->new_public_xtra),
+                             (void*)(pf_module->free_public_xtra),
+                             (void*)(pf_module->sizeof_temp_data),
+                             (void*)(pf_module->output),
+                             (void*)(pf_module->output_static),
+                             PFModuleInstanceXtra(pf_module),
+                             PFModulePublicXtra(pf_module));
 }
 
 
diff --git a/pfsimulator/parflow_lib/pf_module.h b/pfsimulator/parflow_lib/pf_module.h
index c726fc7e3..2f796e896 100644
--- a/pfsimulator/parflow_lib/pf_module.h
+++ b/pfsimulator/parflow_lib/pf_module.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * PFModule structure and accessor macros
 *
@@ -50,13 +50,13 @@
  * invokes the algorithm.  It is a little like a functor in C++.  This
  * is limited so two Output methods have been added in Extended
  * modules.  And yes, moving to C++ would make this cleaner.
- * 
- * Construction is broken down into two calls.   PublicExtra is data that is 
+ *
+ * Construction is broken down into two calls.   PublicExtra is data that is
  * shared by all instances of that type of module.   The Extra concept is
- * much like a static member of a C++ class.   InstanceExtra is the 
+ * much like a static member of a C++ class.   InstanceExtra is the
  * data that is unique to each instance of an algorithm.
  *
- * The output methods can be called to output state that is 
+ * The output methods can be called to output state that is
  * time variant (called at each timestep as indicated by the user)
  * or time invariant (called only once).
  */
@@ -68,12 +68,12 @@ typedef struct {
 
   /**
    * Virtual method invoked to construct instance data.
-   * 
-   * Instance data is per instantiation the module type; it is 
+   *
+   * Instance data is per instantiation the module type; it is
    * not shared between instances.
    */
   void (*init_instance_xtra)();
-  
+
   /**
    * Virtual method invoked to free the instance data.
    */
@@ -91,11 +91,11 @@ typedef struct {
    * Virtual method invoked to free the class data.
    */
   void (*free_public_xtra)();
-  
+
   /**
    * Virtual method to compute the size of the temporary data.
    *
-   * This returns the size of the temporary data used by 
+   * This returns the size of the temporary data used by
    * the instance.
    */
   int (*sizeof_temp_data)();
@@ -132,7 +132,7 @@ typedef struct {
 
 /**
  * A 'this' pointer for module methods.
- * 
+ *
  * Used by module macros below to pass module xtra data to module
  * routines This is similar in purpose to the 'this' pointer in C++.
  * Rather than passing this in each function it is done through a
@@ -151,13 +151,13 @@ amps_ThreadLocalDcl(extern PFModule *, global_ptr_this_pf_module);
 
 /**
  * Device copy of the 'this' module pointer.
- */ 
+ */
 #if (PARFLOW_ACC_BACKEND == PARFLOW_BACKEND_CUDA) && defined(__CUDACC__)
 #ifdef PARFLOW_GLOBALS
 __device__ PFModule *dev_global_ptr_this_pf_module;
 #else
-/* This extern requires CUDA separate compilation, otherwise nvcc compiler 
-   treats the pointer as static variable for each compilation unit          */
+/* This extern requires CUDA separate compilation, otherwise nvcc compiler
+ * treats the pointer as static variable for each compilation unit          */
 extern __device__ PFModule *dev_global_ptr_this_pf_module;
 #endif // PARFLOW_GLOBALS
 #endif // PARFLOW_ACC_BACKEND == PARFLOW_BACKEND_CUDA && __CUDACC__
@@ -200,7 +200,7 @@ extern __device__ PFModule *dev_global_ptr_this_pf_module;
  *
  * Modules are based on the idea the module has single method that
  * does the work of the algorithm.  A bit like the functor pattern in
- * C++.   
+ * C++.
  *
  * @note
  * Rational: This was done so all PFModules would have the same
@@ -211,11 +211,11 @@ extern __device__ PFModule *dev_global_ptr_this_pf_module;
  * @param args Arguments for the method to invoke.
  * @return The invoked method return
  */
-#define PFModuleInvokeType(type, pf_module, args) \
-  (                                               \
-   ThisPFModule = pf_module,                      \
-   (*(type)(ThisPFModule->call))args              \
-  )
+#define PFModuleInvokeType(type, pf_module, args)        \
+        (                                                \
+         ThisPFModule = pf_module,                       \
+         (*(type)(ThisPFModule->call)) args              \
+        )
 
 /**
  * Create a new instance of a module.
@@ -223,21 +223,21 @@ extern __device__ PFModule *dev_global_ptr_this_pf_module;
  * Note we have done some renaming to better match OO (C++) naming
  * schemes.  This invokes the "init" method in the method which is a
  * ctor for the method instance.
- * 
+ *
  * The method invoked is assumed to be of type 'PFModule * (*)()'.
  * The PFModuleNewInstanceType can be used to specify the actual type
  * of the method to invoke.  It is better to specify the type; PF
  * started with KR style C with less type safety.
- * 
+ *
  * @param pf_module The module instance
  * @param args Arguments for the method to invoke
  * @return The new module instance pointer
  */
-#define PFModuleNewInstance(pf_module, args)                   \
-  (                                                            \
-   ThisPFModule = DupPFModule(pf_module),                      \
-   (*(PFModule * (*)())(ThisPFModule->init_instance_xtra))args \
-  )
+#define PFModuleNewInstance(pf_module, args)                          \
+        (                                                             \
+         ThisPFModule = DupPFModule(pf_module),                       \
+         (*(PFModule * (*)())(ThisPFModule->init_instance_xtra)) args \
+        )
 
 /**
  * Create a new instance of a module.
@@ -251,15 +251,15 @@ extern __device__ PFModule *dev_global_ptr_this_pf_module;
  * @param args Arguments for the method to invoke
  * @return The new module instance pointer
  */
-#define PFModuleNewInstanceType(type, pf_module, args) \
-  (                                                    \
-   ThisPFModule = DupPFModule(pf_module),              \
-   (*(type)(ThisPFModule->init_instance_xtra))args     \
-  )
+#define PFModuleNewInstanceType(type, pf_module, args)        \
+        (                                                     \
+         ThisPFModule = DupPFModule(pf_module),               \
+         (*(type)(ThisPFModule->init_instance_xtra)) args     \
+        )
 
 /**
  * 'ReNew' the instance of a module.
- * 
+ *
  * 'ReNew'ing a module is done when the module has already been
  * constructed but some state may have changed.
  *
@@ -267,7 +267,7 @@ extern __device__ PFModule *dev_global_ptr_this_pf_module;
  * The PFModuleReNewInstanceType can be used to specify the actual
  * type of the method to invoke.  It is better to specify the type; PF
  * started with KR style C with less type safety.
- * 
+ *
  * \TODO this should be better documented.  What do we mean by rewnew
  * and when/how is it used.
  *
@@ -275,18 +275,18 @@ extern __device__ PFModule *dev_global_ptr_this_pf_module;
  * @param args Arguments for the method to invoke
  * @return The new module instance pointer
  */
-#define PFModuleReNewInstance(pf_module, args)                 \
-  (                                                            \
-   ThisPFModule = pf_module,                                   \
-   (*(PFModule * (*)())(ThisPFModule->init_instance_xtra))args \
-  )
+#define PFModuleReNewInstance(pf_module, args)                        \
+        (                                                             \
+         ThisPFModule = pf_module,                                    \
+         (*(PFModule * (*)())(ThisPFModule->init_instance_xtra)) args \
+        )
 
 /**
  * 'ReNew' the instance of a module.
- * 
+ *
  * 'ReNew'ing a module is done when the module has already been
  * constructed but some state may have changed.
- * 
+ *
  * \TODO this should be better documented.  What do we mean by rewnew
  * and when/how is it used.
  *
@@ -294,11 +294,11 @@ extern __device__ PFModule *dev_global_ptr_this_pf_module;
  * @param args Arguments for the method to invoke.
  * @return The new module instance pointer.
  */
-#define PFModuleReNewInstanceType(type, pf_module, args) \
-  (                                                      \
-   ThisPFModule = pf_module,                             \
-   (*(type)(ThisPFModule->init_instance_xtra))args       \
-  )
+#define PFModuleReNewInstanceType(type, pf_module, args)        \
+        (                                                       \
+         ThisPFModule = pf_module,                              \
+         (*(type)(ThisPFModule->init_instance_xtra)) args       \
+        )
 
 /**
  * Free the module.
@@ -307,16 +307,16 @@ extern __device__ PFModule *dev_global_ptr_this_pf_module;
  *
  * @param pf_module The module instance
  */
-#define PFModuleFreeInstance(pf_module)                 \
-  (                                                     \
-   ThisPFModule = pf_module,                            \
-   (*(void (*)())(ThisPFModule->free_instance_xtra))(), \
-   FreePFModule(pf_module)                              \
-  )
+#define PFModuleFreeInstance(pf_module)                       \
+        (                                                     \
+         ThisPFModule = pf_module,                            \
+         (*(void (*)())(ThisPFModule->free_instance_xtra))(), \
+         FreePFModule(pf_module)                              \
+        )
 
 /**
  * Output time variant state associated with this module instance.
- * 
+ *
  * This method is invoked at the output time intervals specified by
  * the user input.  The method should write any vectors or other state
  * out for the current timestep.
@@ -325,16 +325,16 @@ extern __device__ PFModule *dev_global_ptr_this_pf_module;
  * @param pf_module The module instance
  * @param args Arguments for the output method
  */
-#define PFModuleOutputType(type, pf_module, args) \
-  (                                           \
-   ThisPFModule = pf_module,                  \
-   (*(type (*)())(ThisPFModule->output))args  \
-  )
+#define PFModuleOutputType(type, pf_module, args)    \
+        (                                            \
+         ThisPFModule = pf_module,                   \
+         (*(type (*)())(ThisPFModule->output)) args  \
+        )
 
 
 /**
  * Output static (time invariant) state associated with this module instance.
- * 
+ *
  * This method is invoked at the start of a run and should write any
  * vectors or other state for the module that does not require a
  * timestamp.
@@ -343,11 +343,11 @@ extern __device__ PFModule *dev_global_ptr_this_pf_module;
  * @param pf_module The module instance
  * @param args Arguments for the output method
  */
-#define PFModuleOutputStaticType(type, pf_module, args) \
-  (                                           \
-   ThisPFModule = pf_module,                  \
-   (*(type (*)(char file_prefix[2048],ProblemData *))(ThisPFModule->output_static))args	\
-  )
+#define PFModuleOutputStaticType(type, pf_module, args)                                         \
+        (                                                                                       \
+         ThisPFModule = pf_module,                                                              \
+         (*(type (*)(char file_prefix[2048], ProblemData *))(ThisPFModule->output_static)) args \
+        )
 
 /**
  * Create a class of module.
@@ -359,17 +359,17 @@ extern __device__ PFModule *dev_global_ptr_this_pf_module;
  * @param args Arguments for the module class constructor
  * @return The new module instance
  */
-#define PFModuleNewModule(name, args)                          \
-  (                                                            \
-   ThisPFModule = NewPFModule((void*)name,                     \
-                              (void*)name ## InitInstanceXtra, \
-                              (void*)name ## FreeInstanceXtra, \
-                              (void*)name ## NewPublicXtra,    \
-                              (void*)name ## FreePublicXtra,   \
-                              (void*)name ## SizeOfTempData,   \
-                              NULL, NULL),                     \
-   (*(PFModule * (*)())(ThisPFModule->new_public_xtra))args    \
-  )
+#define PFModuleNewModule(name, args)                                 \
+        (                                                             \
+         ThisPFModule = NewPFModule((void*)name,                      \
+                                    (void*)name ## InitInstanceXtra,  \
+                                    (void*)name ## FreeInstanceXtra,  \
+                                    (void*)name ## NewPublicXtra,     \
+                                    (void*)name ## FreePublicXtra,    \
+                                    (void*)name ## SizeOfTempData,    \
+                                    NULL, NULL),                      \
+         (*(PFModule * (*)())(ThisPFModule->new_public_xtra)) args    \
+        )
 
 /**
  * Create a class of module.
@@ -382,21 +382,21 @@ extern __device__ PFModule *dev_global_ptr_this_pf_module;
  * @param args Arguments for the module class constructor
  * @return The new module instance
  */
-#define PFModuleNewModuleType(type, name, args)                \
-  (                                                            \
-   ThisPFModule = NewPFModule((void*)name,                     \
-                              (void*)name ## InitInstanceXtra, \
-                              (void*)name ## FreeInstanceXtra, \
-                              (void*)name ## NewPublicXtra,    \
-                              (void*)name ## FreePublicXtra,   \
-                              (void*)name ## SizeOfTempData,   \
-                              NULL, NULL),                     \
-   (*(type)(ThisPFModule->new_public_xtra))args                \
-  )
+#define PFModuleNewModuleType(type, name, args)                       \
+        (                                                             \
+         ThisPFModule = NewPFModule((void*)name,                      \
+                                    (void*)name ## InitInstanceXtra,  \
+                                    (void*)name ## FreeInstanceXtra,  \
+                                    (void*)name ## NewPublicXtra,     \
+                                    (void*)name ## FreePublicXtra,    \
+                                    (void*)name ## SizeOfTempData,    \
+                                    NULL, NULL),                      \
+         (*(type)(ThisPFModule->new_public_xtra)) args                \
+        )
 
 /**
  * Create a class of extended module.
- * 
+ *
  * For use with modules that implement the Extended module API.
  * Currently this is the Output methods.
  *
@@ -408,32 +408,32 @@ extern __device__ PFModule *dev_global_ptr_this_pf_module;
  * @param args Arguments for the module class constructor
  * @return The new module instance
  */
-#define PFModuleNewModuleExtendedType(type, name, args)        \
-  (                                                            \
-   ThisPFModule = NewPFModuleExtended((void*)name,                     \
-                              (void*)name ## InitInstanceXtra, \
-                              (void*)name ## FreeInstanceXtra, \
-                              (void*)name ## NewPublicXtra,    \
-                              (void*)name ## FreePublicXtra,   \
-                              (void*)name ## SizeOfTempData,   \
-                              (void*)name ## Output,	       \
-                              (void*)name ## OutputStatic,     \
-                              NULL, NULL),                     \
-   (*(type)(ThisPFModule->new_public_xtra))args                \
-  )
+#define PFModuleNewModuleExtendedType(type, name, args)                      \
+        (                                                                    \
+         ThisPFModule = NewPFModuleExtended((void*)name,                     \
+                                            (void*)name ## InitInstanceXtra, \
+                                            (void*)name ## FreeInstanceXtra, \
+                                            (void*)name ## NewPublicXtra,    \
+                                            (void*)name ## FreePublicXtra,   \
+                                            (void*)name ## SizeOfTempData,   \
+                                            (void*)name ## Output,           \
+                                            (void*)name ## OutputStatic,     \
+                                            NULL, NULL),                     \
+         (*(type)(ThisPFModule->new_public_xtra)) args                       \
+        )
 
 
 /**
  * Invoke the destructor of the module.
- * 
+ *
  * @param pf_module The module instance
  */
-#define PFModuleFreeModule(pf_module)                 \
-  (                                                   \
-   ThisPFModule = pf_module,                          \
-   (*(void (*)())(ThisPFModule->free_public_xtra))(), \
-   FreePFModule(pf_module)                            \
-  )
+#define PFModuleFreeModule(pf_module)                       \
+        (                                                   \
+         ThisPFModule = pf_module,                          \
+         (*(void (*)())(ThisPFModule->free_public_xtra))(), \
+         FreePFModule(pf_module)                            \
+        )
 
 /**
  * Return this size of the temporary data needed by this module instance.
@@ -441,22 +441,22 @@ extern __device__ PFModule *dev_global_ptr_this_pf_module;
  * Temp date is a block of memory allocated in the solver and modules
  * use this block.  Data use may be overlayed in this block.  The
  * size of the block is the high water mark.
- * 
+ *
  * Use of temp data has mostly been removed but some modules
  * (e.g. advection_godunov) still use temp_data.  Modern memory
  * allocators have removed much of the need for this mechanism.
  *
  * @note
  * Rational: PF originally operated on very low memory compute nodes
- * (4MB-16MB) so had to carefully manage space for vectors.   This 
+ * (4MB-16MB) so had to carefully manage space for vectors.   This
  * method is part of that mechanism.
  *
  * @param pf_module The module instance
  */
-#define PFModuleSizeOfTempData(pf_module)           \
-  (                                                 \
-   ThisPFModule = pf_module,                        \
-   (*(int (*)())(ThisPFModule->sizeof_temp_data))() \
-  )
+#define PFModuleSizeOfTempData(pf_module)                 \
+        (                                                 \
+         ThisPFModule = pf_module,                        \
+         (*(int (*)())(ThisPFModule->sizeof_temp_data))() \
+        )
 
 #endif
diff --git a/pfsimulator/parflow_lib/pf_omploops.h b/pfsimulator/parflow_lib/pf_omploops.h
index f4167eabe..19fe5bdd2 100644
--- a/pfsimulator/parflow_lib/pf_omploops.h
+++ b/pfsimulator/parflow_lib/pf_omploops.h
@@ -1,23 +1,32 @@
-/*BHEADER*********************************************************************
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
-
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
+
+/* *INDENT-OFF* */
 #ifndef _PF_OMPLOOPS_H_
 #define _PF_OMPLOOPS_H_
 
@@ -713,3 +722,4 @@ static const int FDIR_TABLE[6][3] = {
 
 #endif // PARFLOW_HAVE_OMP
 #endif // _PF_OMPLOOPS_H_
+/* *INDENT-ON* */
diff --git a/pfsimulator/parflow_lib/pf_pfmg.c b/pfsimulator/parflow_lib/pf_pfmg.c
index 0efa2596d..3febab14c 100644
--- a/pfsimulator/parflow_lib/pf_pfmg.c
+++ b/pfsimulator/parflow_lib/pf_pfmg.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -185,21 +185,21 @@ PFModule  *PFMGInitInstanceXtra(
     }
 
     HypreInitialize(pf_Bmat,
-		    &(instance_xtra -> hypre_grid),
-		    &(instance_xtra -> hypre_stencil),
-		    &(instance_xtra -> hypre_mat),
-		    &(instance_xtra -> hypre_b),
-		    &(instance_xtra -> hypre_x)
-		    );
+                    &(instance_xtra->hypre_grid),
+                    &(instance_xtra->hypre_stencil),
+                    &(instance_xtra->hypre_mat),
+                    &(instance_xtra->hypre_b),
+                    &(instance_xtra->hypre_x)
+                    );
 
     /* Copy the matrix entries */
     BeginTiming(public_xtra->time_index_copy_hypre);
 
     HypreAssembleMatrixAsElements(pf_Bmat,
-				  pf_Cmat,
-				  &(instance_xtra -> hypre_mat),
-				  problem_data);
-    
+                                  pf_Cmat,
+                                  &(instance_xtra->hypre_mat),
+                                  problem_data);
+
     EndTiming(public_xtra->time_index_copy_hypre);
 
     /* Set up the PFMG preconditioner */
@@ -307,7 +307,7 @@ PFModule  *PFMGNewPublicXtra(char *name)
   public_xtra->raptype = NA_NameToIndexExitOnError(raptype_switch_na, raptype_name, key);
   NA_FreeNameArray(raptype_switch_na);
 
-  if (public_xtra->raptype == 0 && public_xtra->smoother  > 1)
+  if (public_xtra->raptype == 0 && public_xtra->smoother > 1)
   {
     InputError("Error: Galerkin RAPType is not compatible with Smoother <%s>.\n",
                smoother_name, key);
diff --git a/pfsimulator/parflow_lib/pf_pfmg_octree.c b/pfsimulator/parflow_lib/pf_pfmg_octree.c
index a57ac78bb..0d0a1bd76 100644
--- a/pfsimulator/parflow_lib/pf_pfmg_octree.c
+++ b/pfsimulator/parflow_lib/pf_pfmg_octree.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -843,15 +843,15 @@ PFModule  *PFMGOctreeInitInstanceXtra(
             hypre_BoxDestroy(set_box);
           });
 
-	  hypre_BoxDestroy(value_box);
+          hypre_BoxDestroy(value_box);
         }
       }     /* End subgrid loop */
 
       ForSubgridI(sg, GridSubgrids(mat_grid))
       {
-	double *wp = NULL, *ep, *sop = NULL, *np, *lp = NULL, *up = NULL;
-	double *cp_c, *wp_c = NULL, *ep_c = NULL, *sop_c = NULL, *np_c = NULL;
-	
+        double *wp = NULL, *ep, *sop = NULL, *np, *lp = NULL, *up = NULL;
+        double *cp_c, *wp_c = NULL, *ep_c = NULL, *sop_c = NULL, *np_c = NULL;
+
         subgrid = GridSubgrid(mat_grid, sg);
 
         pfB_sub = MatrixSubmatrix(pf_Bmat, sg);
@@ -902,30 +902,30 @@ PFModule  *PFMGOctreeInitInstanceXtra(
         ny_m = SubmatrixNY(pfB_sub);
         nz_m = SubmatrixNZ(pfB_sub);
 
-	int sy_v = SubvectorNX(top_sub);
+        int sy_v = SubvectorNX(top_sub);
 
         im = SubmatrixEltIndex(pfB_sub, ix, iy, iz);
 
         if (symmetric)
         {
-	  /* 
-	   * Loop in 2D over top points for surface contributions
-	   * We need to loop separately over this since the above
+          /*
+           * Loop in 2D over top points for surface contributions
+           * We need to loop separately over this since the above
            * box loop does not allow us to loop over the individual
            * top cells. For nonsymmetric, we need to include the
            * diagonal, east, west, north, and south terms - DOK
-	   */
+           */
           BoxLoopI1(i, j, k, ix, iy, 0, nx, ny, 1,
                     im, nx_m, ny_m, nz_m, 1, 1, 1,
           {
             itop = SubvectorEltIndex(top_sub, i, j, 0);
             ktop = (int)top_dat[itop];
 
-	    if (ktop >= 0)
+            if (ktop >= 0)
             {
-	      io = SubmatrixEltIndex(pfC_sub, i, j, 0);
-	      int ioB = SubmatrixEltIndex(pfB_sub, i, j, ktop);
-	      
+              io = SubmatrixEltIndex(pfC_sub, i, j, 0);
+              int ioB = SubmatrixEltIndex(pfB_sub, i, j, ktop);
+
               /* update diagonal coeff */
               coeffs_symm[0] = cp_c[io];               //cp[im] is zero
               /* update east coeff */
@@ -934,39 +934,38 @@ PFModule  *PFMGOctreeInitInstanceXtra(
               coeffs_symm[2] = np[ioB];
               /* update upper coeff */
               coeffs_symm[3] = up[ioB];               // JB keeps upper term on surface. This should be zero
-	      
-	      index[0] = i;
-	      index[1] = j;
-	      index[2] = ktop;
-	      HYPRE_StructMatrixSetValues(instance_xtra->hypre_mat,
-					  index,
-					  stencil_size,
-					  stencil_indices_symm,
-					  coeffs_symm);
-	    }
-	    
-	  }); // BoxLoop
-        } // symmetric 
+
+              index[0] = i;
+              index[1] = j;
+              index[2] = ktop;
+              HYPRE_StructMatrixSetValues(instance_xtra->hypre_mat,
+                                          index,
+                                          stencil_size,
+                                          stencil_indices_symm,
+                                          coeffs_symm);
+            }
+          }); // BoxLoop
+        } // symmetric
         else
         {
-	  /* 
-	   * Loop in 2D over top points for surface contributions
-	   * We need to loop separately over this since the above
+          /*
+           * Loop in 2D over top points for surface contributions
+           * We need to loop separately over this since the above
            * box loop does not allow us to loop over the individual
            * top cells. For nonsymmetric, we need to include the
            * diagonal, east, west, north, and south terms - DOK
-	   */
+           */
           BoxLoopI1(i, j, k, ix, iy, 0, nx, ny, 1,
                     im, nx_m, ny_m, nz_m, 1, 1, 1,
           {
             itop = SubvectorEltIndex(top_sub, i, j, 0);
             ktop = (int)top_dat[itop];
 
-	    if (ktop >= 0)
+            if (ktop >= 0)
             {
-	      io = SubmatrixEltIndex(pfC_sub, i, j, 0);
-	      int ioB = SubmatrixEltIndex(pfB_sub, i, j, ktop);
-	      
+              io = SubmatrixEltIndex(pfC_sub, i, j, 0);
+              int ioB = SubmatrixEltIndex(pfB_sub, i, j, ktop);
+
               /* update diagonal coeff */
               coeffs[0] = cp_c[io];               //cp[ioB] is zero
               /* update west coeff */
@@ -998,19 +997,17 @@ PFModule  *PFMGOctreeInitInstanceXtra(
               /* update upper coeff */
               coeffs[6] = up[ioB];               // JB keeps upper term on surface. This should be zero
 
-	      index[0] = i;
-	      index[1] = j;
-	      index[2] = ktop;
-	      HYPRE_StructMatrixSetValues(instance_xtra->hypre_mat,
-					  index,
-					  stencil_size,
-					  stencil_indices, coeffs);
-	    }
-
-	  }); // BoxLoop
+              index[0] = i;
+              index[1] = j;
+              index[2] = ktop;
+              HYPRE_StructMatrixSetValues(instance_xtra->hypre_mat,
+                                          index,
+                                          stencil_size,
+                                          stencil_indices, coeffs);
+            }
+          }); // BoxLoop
         } // non symmetric
       }   /* End subgrid loop */
-      
     }  /* end if pf_Cmat==NULL */
 
 
@@ -1109,7 +1106,7 @@ PFModule  *PFMGOctreeNewPublicXtra(char *name)
   smoother_switch_na = NA_NewNameArray("Jacobi WJacobi RBGaussSeidelSymmetric RBGaussSeidelNonSymmetric");
   sprintf(key, "%s.Smoother", name);
   smoother_name = GetStringDefault(key, "RBGaussSeidelNonSymmetric");
-  public_xtra -> smoother = NA_NameToIndexExitOnError(smoother_switch_na, smoother_name, key);
+  public_xtra->smoother = NA_NameToIndexExitOnError(smoother_switch_na, smoother_name, key);
   NA_FreeNameArray(smoother_switch_na);
 
   public_xtra->time_index_pfmg = RegisterTiming("PFMGOctree");
diff --git a/pfsimulator/parflow_lib/pf_smg.c b/pfsimulator/parflow_lib/pf_smg.c
index c61a02c98..498d29b4a 100644
--- a/pfsimulator/parflow_lib/pf_smg.c
+++ b/pfsimulator/parflow_lib/pf_smg.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -141,8 +141,8 @@ PFModule  *SMGInitInstanceXtra(
                                Problem *    problem,
                                Grid *       grid,
                                ProblemData *problem_data,
-			       Matrix *     pf_Bmat,
-			       Matrix *     pf_Cmat,
+                               Matrix *     pf_Bmat,
+                               Matrix *     pf_Cmat,
                                double *     temp_data)
 {
 #ifdef HAVE_HYPRE
@@ -181,20 +181,20 @@ PFModule  *SMGInitInstanceXtra(
 
 
     HypreInitialize(pf_Bmat,
-		    &(instance_xtra -> hypre_grid),
-		    &(instance_xtra -> hypre_stencil),
-		    &(instance_xtra -> hypre_mat),
-		    &(instance_xtra -> hypre_b),
-		    &(instance_xtra -> hypre_x)
-		    );
+                    &(instance_xtra->hypre_grid),
+                    &(instance_xtra->hypre_stencil),
+                    &(instance_xtra->hypre_mat),
+                    &(instance_xtra->hypre_b),
+                    &(instance_xtra->hypre_x)
+                    );
 
     /* Copy the matrix entries */
     BeginTiming(public_xtra->time_index_copy_hypre);
-    
+
     HypreAssembleMatrixAsElements(pf_Bmat,
-				  pf_Cmat,
-				  &(instance_xtra -> hypre_mat),
-				  problem_data);
+                                  pf_Cmat,
+                                  &(instance_xtra->hypre_mat),
+                                  problem_data);
 
     EndTiming(public_xtra->time_index_copy_hypre);
 
diff --git a/pfsimulator/parflow_lib/pfield.c b/pfsimulator/parflow_lib/pfield.c
index 2852083c1..60bcb09ea 100644
--- a/pfsimulator/parflow_lib/pfield.c
+++ b/pfsimulator/parflow_lib/pfield.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
diff --git a/pfsimulator/parflow_lib/pgsRF.c b/pfsimulator/parflow_lib/pgsRF.c
index 501667469..f5eaf66ba 100644
--- a/pfsimulator/parflow_lib/pgsRF.c
+++ b/pfsimulator/parflow_lib/pgsRF.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
diff --git a/pfsimulator/parflow_lib/phase_velocity_face.c b/pfsimulator/parflow_lib/phase_velocity_face.c
index a5cacf734..a64de1d7f 100644
--- a/pfsimulator/parflow_lib/phase_velocity_face.c
+++ b/pfsimulator/parflow_lib/phase_velocity_face.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *****************************************************************************/
@@ -424,7 +424,7 @@ void          PhaseVelocityFace(
    * --------------------------------------------------------------------
    * --------------------------------------------------------------------
    * Fixed global boundary values for saturated solver - JJB 04/21
-   * 
+   *
    *----------------------------------------------------------------------*/
 
   bc_struct = PFModuleInvokeType(BCPressureInvoke, bc_pressure,
@@ -481,122 +481,121 @@ void          PhaseVelocityFace(
                            Locals(int ip, vx_l, vy_l, vz_l;
                                   int alpha, vel_idx;
                                   double *mob_vec, *vel_vec;
-                                  double pdiff, value, vel_h;),
+                                  double pdiff, value, vel_h; ),
                            CellSetup({
-                               ip = SubvectorEltIndex(p_sub, i, j, k);
+        ip = SubvectorEltIndex(p_sub, i, j, k);
 
-                               vx_l = SubvectorEltIndex(vx_sub, i, j, k);
-                               vy_l = SubvectorEltIndex(vy_sub, i, j, k);
-                               vz_l = SubvectorEltIndex(vz_sub, i, j, k);
+        vx_l = SubvectorEltIndex(vx_sub, i, j, k);
+        vy_l = SubvectorEltIndex(vy_sub, i, j, k);
+        vz_l = SubvectorEltIndex(vz_sub, i, j, k);
 
-                               alpha = 0;
-                               vel_idx = 0;
+        alpha = 0;
+        vel_idx = 0;
 
-                               mob_vec = NULL;
-                               vel_vec = NULL;
+        mob_vec = NULL;
+        vel_vec = NULL;
 
-                               vel_h = 0.0e0;
-                               pdiff = 0.0e0;
-                               value = bc_patch_values[ival];
-                             }),
+        vel_h = 0.0e0;
+        pdiff = 0.0e0;
+        value = bc_patch_values[ival];
+      }),
                            FACE(LeftFace, {
-                               alpha = 0;
-                               mob_vec = mx;
-                               vel_vec = vx;
-                               vel_h = dx;
-                               vel_idx = vx_l;
-                               pdiff = value - pres[ip];
-                             }),
+        alpha = 0;
+        mob_vec = mx;
+        vel_vec = vx;
+        vel_h = dx;
+        vel_idx = vx_l;
+        pdiff = value - pres[ip];
+      }),
                            FACE(RightFace, {
-                               alpha = 0;
-                               mob_vec = mx;
-                               vel_vec = vx;
-                               vel_h = dx;
-                               vel_idx = vx_l + 1;
-                               pdiff = pres[ip] - value;
-                             }),
+        alpha = 0;
+        mob_vec = mx;
+        vel_vec = vx;
+        vel_h = dx;
+        vel_idx = vx_l + 1;
+        pdiff = pres[ip] - value;
+      }),
                            FACE(DownFace, {
-                               alpha = 0;
-                               mob_vec = my;
-                               vel_vec = vy;
-                               vel_h = dy;
-                               vel_idx = vy_l;
-                               pdiff = value - pres[ip];
-                             }),
+        alpha = 0;
+        mob_vec = my;
+        vel_vec = vy;
+        vel_h = dy;
+        vel_idx = vy_l;
+        pdiff = value - pres[ip];
+      }),
                            FACE(UpFace, {
-                               alpha = 0;
-                               mob_vec = my;
-                               vel_vec = vy;
-                               vel_h = dy;
-                               vel_idx = vy_l + sy_v;
-                               pdiff = pres[ip] - value;
-                             }),
+        alpha = 0;
+        mob_vec = my;
+        vel_vec = vy;
+        vel_h = dy;
+        vel_idx = vy_l + sy_v;
+        pdiff = pres[ip] - value;
+      }),
                            FACE(BackFace, {
-                               alpha = 1;
-                               mob_vec = mz;
-                               vel_vec = vz;
-                               vel_h = dz;
-                               vel_idx = vz_l;
-                               pdiff = value - pres[ip];
-                             }),
+        alpha = 1;
+        mob_vec = mz;
+        vel_vec = vz;
+        vel_h = dz;
+        vel_idx = vz_l;
+        pdiff = value - pres[ip];
+      }),
                            FACE(FrontFace, {
-                               alpha = -1;
-                               mob_vec = mz;
-                               vel_vec = vz;
-                               vel_h = dz;
-                               vel_idx = vz_l + sz_v;
-                               pdiff = pres[ip] - value;
-                             }),
+        alpha = -1;
+        mob_vec = mz;
+        vel_vec = vz;
+        vel_h = dz;
+        vel_idx = vz_l + sz_v;
+        pdiff = pres[ip] - value;
+      }),
                            CellFinalize({
-                               vel_vec[vel_idx] = mob_vec[ip] * (pdiff / (0.5 * vel_h) - alpha * den[ip] * ProblemGravity(problem));
-                             }),
+        vel_vec[vel_idx] = mob_vec[ip] * (pdiff / (0.5 * vel_h) - alpha * den[ip] * ProblemGravity(problem));
+      }),
                            AfterAllCells(DoNothing)
-        );
+                           );
 
       ForPatchCellsPerFace(FluxBC,
                            BeforeAllCells(DoNothing),
                            LoopVars(i, j, k, ival, bc_struct, ipatch, is),
                            Locals(int vel_idx, vx_l, vy_l, vz_l;
                                   double *vel_vec;
-                                  double value;),
+                                  double value; ),
                            CellSetup({
+        vx_l = SubvectorEltIndex(vx_sub, i, j, k);
+        vy_l = SubvectorEltIndex(vy_sub, i, j, k);
+        vz_l = SubvectorEltIndex(vz_sub, i, j, k);
 
-                               vx_l = SubvectorEltIndex(vx_sub, i, j, k);
-                               vy_l = SubvectorEltIndex(vy_sub, i, j, k);
-                               vz_l = SubvectorEltIndex(vz_sub, i, j, k);
-
-                               vel_idx = 0;
-                               vel_vec = NULL;
-
-                               value = bc_patch_values[ival];
-                             }),
-                           FACE(LeftFace,  { 
-                               vel_vec = vx;
-                               vel_idx = vx_l;
-                             }),
-                           FACE(RightFace, { 
-                               vel_vec = vx;
-                               vel_idx = vx_l + 1;
-                             }),
-                           FACE(DownFace,  { 
-                               vel_vec = vy;
-                               vel_idx = vy_l;
-                             }),
-                           FACE(UpFace,    { 
-                               vel_vec = vy;
-                               vel_idx = vy_l + sy_v;
-                             }),
-                           FACE(BackFace,  { 
-                               vel_vec = vz;
-                               vel_idx = vz_l;
-                             }),
-                           FACE(FrontFace, { 
-                               vel_vec = vz;
-                               vel_idx = vz_l + sz_v;
-                             }),
+        vel_idx = 0;
+        vel_vec = NULL;
+
+        value = bc_patch_values[ival];
+      }),
+                           FACE(LeftFace, {
+        vel_vec = vx;
+        vel_idx = vx_l;
+      }),
+                           FACE(RightFace, {
+        vel_vec = vx;
+        vel_idx = vx_l + 1;
+      }),
+                           FACE(DownFace, {
+        vel_vec = vy;
+        vel_idx = vy_l;
+      }),
+                           FACE(UpFace, {
+        vel_vec = vy;
+        vel_idx = vy_l + sy_v;
+      }),
+                           FACE(BackFace, {
+        vel_vec = vz;
+        vel_idx = vz_l;
+      }),
+                           FACE(FrontFace, {
+        vel_vec = vz;
+        vel_idx = vz_l + sz_v;
+      }),
                            CellFinalize({ vel_vec[vel_idx] = value; }),
                            AfterAllCells(DoNothing)
-        );
+                           );
     }
   }
 
@@ -783,7 +782,7 @@ void PhaseVelocityFaceFreePublicXtra()
 
   if (public_xtra)
   {
-    free(public_xtra);
+    tfree(public_xtra);
   }
 }
 
diff --git a/pfsimulator/parflow_lib/ppcg.c b/pfsimulator/parflow_lib/ppcg.c
index f5b795136..bbb313960 100644
--- a/pfsimulator/parflow_lib/ppcg.c
+++ b/pfsimulator/parflow_lib/ppcg.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Chebyshev polynomial preconditioned conjugate gradient solver (Omin).
diff --git a/pfsimulator/parflow_lib/printgrid.c b/pfsimulator/parflow_lib/printgrid.c
index dde643466..f3abe7531 100644
--- a/pfsimulator/parflow_lib/printgrid.c
+++ b/pfsimulator/parflow_lib/printgrid.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Print the grid structure to a file
diff --git a/pfsimulator/parflow_lib/printmatrix.c b/pfsimulator/parflow_lib/printmatrix.c
index 359f94511..553747dfb 100644
--- a/pfsimulator/parflow_lib/printmatrix.c
+++ b/pfsimulator/parflow_lib/printmatrix.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routines for printing matrices to file
diff --git a/pfsimulator/parflow_lib/printvector.c b/pfsimulator/parflow_lib/printvector.c
index e5d7f646c..5ad34f055 100644
--- a/pfsimulator/parflow_lib/printvector.c
+++ b/pfsimulator/parflow_lib/printvector.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routines for printing vectors to file
diff --git a/pfsimulator/parflow_lib/problem.c b/pfsimulator/parflow_lib/problem.c
index 72f4916fd..23bd1f16b 100644
--- a/pfsimulator/parflow_lib/problem.c
+++ b/pfsimulator/parflow_lib/problem.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
@@ -210,6 +210,12 @@ Problem   *NewProblem(
   ProblemYSlope(problem) =
     PFModuleNewModule(YSlope, ());   //sk
 
+  ProblemXChannelWidth(problem) =
+    PFModuleNewModule(XChannelWidth, ());
+
+  ProblemYChannelWidth(problem) =
+    PFModuleNewModule(YChannelWidth, ());
+
   ProblemMannings(problem) =
     PFModuleNewModule(Mannings, ());   //sk
 
@@ -329,6 +335,9 @@ Problem   *NewProblem(
     PFModuleNewModuleType(WellPackageNewPublicXtraInvoke,
                           WellPackage, (num_phases, num_contaminants));
 
+  ProblemReservoirPackage(problem) =
+    PFModuleNewModuleType(ReservoirPackageNewPublicXtraInvoke,
+                          ReservoirPackage, (num_phases, num_contaminants));
 
   return problem;
 }
@@ -343,6 +352,7 @@ void      FreeProblem(
                       int      solver)
 {
   PFModuleFreeModule(ProblemWellPackage(problem));
+  PFModuleFreeModule(ProblemReservoirPackage(problem));
 
 
   NA_FreeNameArray(GlobalsPhaseNames);
@@ -379,6 +389,8 @@ void      FreeProblem(
   PFModuleFreeModule(ProblemSpecStorage(problem));  //sk
   PFModuleFreeModule(ProblemXSlope(problem));  //sk
   PFModuleFreeModule(ProblemYSlope(problem));
+  PFModuleFreeModule(ProblemXChannelWidth(problem));
+  PFModuleFreeModule(ProblemYChannelWidth(problem));
   PFModuleFreeModule(ProblemMannings(problem));
   PFModuleFreeModule(ProblemdzScale(problem));    //RMM
   PFModuleFreeModule(ProblemFBx(problem));    //RMM
@@ -420,6 +432,8 @@ ProblemData   *NewProblemData(
   ProblemDataSpecificStorage(problem_data) = NewVectorType(grid, 1, 1, vector_cell_centered);  //sk
   ProblemDataTSlopeX(problem_data) = NewVectorType(grid2d, 1, 1, vector_cell_centered_2D);   //sk
   ProblemDataTSlopeY(problem_data) = NewVectorType(grid2d, 1, 1, vector_cell_centered_2D);   //sk
+  ProblemDataChannelWidthX(problem_data) = NewVectorType(grid2d, 1, 1, vector_cell_centered_2D);
+  ProblemDataChannelWidthY(problem_data) = NewVectorType(grid2d, 1, 1, vector_cell_centered_2D);
   ProblemDataMannings(problem_data) = NewVectorType(grid2d, 1, 1, vector_cell_centered_2D);  //sk
 
   /* @RMM added vectors for subsurface slopes for terrain-following grid */
@@ -445,6 +459,7 @@ ProblemData   *NewProblemData(
   ProblemDataBCPressureData(problem_data) = NewBCPressureData();
 
   ProblemDataWellData(problem_data) = NewWellData();
+  ProblemDataReservoirData(problem_data) = NewReservoirData();
 
   return problem_data;
 }
@@ -471,6 +486,7 @@ void          FreeProblemData(
 #endif
 
     FreeWellData(ProblemDataWellData(problem_data));
+    FreeReservoirData(ProblemDataReservoirData(problem_data));
     FreeBCPressureData(ProblemDataBCPressureData(problem_data));
     FreeVector(ProblemDataPorosity(problem_data));
     FreeVector(ProblemDataPermeabilityX(problem_data));
@@ -479,6 +495,8 @@ void          FreeProblemData(
     FreeVector(ProblemDataSpecificStorage(problem_data));   //sk
     FreeVector(ProblemDataTSlopeX(problem_data));   //sk
     FreeVector(ProblemDataTSlopeY(problem_data));   //sk
+    FreeVector(ProblemDataChannelWidthX(problem_data));
+    FreeVector(ProblemDataChannelWidthY(problem_data));
     FreeVector(ProblemDataMannings(problem_data));   //sk
     FreeVector(ProblemDataSSlopeX(problem_data));   //RMM
     FreeVector(ProblemDataSSlopeY(problem_data));   //RMM
@@ -494,3 +512,4 @@ void          FreeProblemData(
     tfree(problem_data);
   }
 }
+
diff --git a/pfsimulator/parflow_lib/problem.h b/pfsimulator/parflow_lib/problem.h
index cb05b1545..c5affc676 100644
--- a/pfsimulator/parflow_lib/problem.h
+++ b/pfsimulator/parflow_lib/problem.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef _PROBLEM_HEADER
 #define _PROBLEM_HEADER
@@ -99,10 +99,13 @@ typedef struct {
 
   /*****  packages  *****/
   PFModule  *well_package;
+  PFModule  *reservoir_package;
 
   /*sk**  overland flow*/
   PFModule  *x_slope;
   PFModule  *y_slope;
+  PFModule  *wc_x;
+  PFModule  *wc_y;
   PFModule  *mann;
   PFModule  *overlandflow_eval;        //DOK
   PFModule  *overlandflow_eval_diff;         //@RMM
@@ -130,7 +133,7 @@ typedef struct {
    */
   Vector         *index_of_domain_top;
 
-  /* 
+  /*
    * This is a NX * NY vector of patch id/index of the top
    * of the domain.
    *
@@ -152,11 +155,14 @@ typedef struct {
 
 
   WellData       *well_data;
+  ReservoirData       *reservoir_data;
   BCPressureData *bc_pressure_data;
 
   /*sk  overland flow*/
   Vector *x_slope;
   Vector *y_slope;
+  Vector *wc_x;
+  Vector *wc_y;
   Vector *mann;
 
   /* @RMM terrain grid */
@@ -218,6 +224,8 @@ typedef struct {
 #define ProblemSpecStorage(problem)               ((problem)->specific_storage)   //sk
 #define ProblemXSlope(problem)                    ((problem)->x_slope)   //sk
 #define ProblemYSlope(problem)                    ((problem)->y_slope)   //sk
+#define ProblemXChannelWidth(problem)             ((problem)->wc_x)
+#define ProblemYChannelWidth(problem)             ((problem)->wc_y)
 #define ProblemFBx(problem)                      ((problem)->FBx)    //RMM
 #define ProblemFBy(problem)                      ((problem)->FBy)    //RMM
 #define ProblemFBz(problem)                      ((problem)->FBz)    //RMM
@@ -246,6 +254,7 @@ typedef struct {
 
 /* packages */
 #define ProblemWellPackage(problem)               ((problem)->well_package)
+#define ProblemReservoirPackage(problem)               ((problem)->reservoir_package)
 
 /* error calculations */
 #define ProblemL2ErrorNorm(problem)               ((problem)->l2_error_norm)
@@ -274,10 +283,13 @@ typedef struct {
 #define ProblemDataFBy(problem_data)            ((problem_data)->FBy)    //RMM
 #define ProblemDataFBz(problem_data)            ((problem_data)->FBz)    //RMM
 #define ProblemDataWellData(problem_data)       ((problem_data)->well_data)
+#define ProblemDataReservoirData(problem_data)       ((problem_data)->reservoir_data)
 #define ProblemDataBCPressureData(problem_data) ((problem_data)->bc_pressure_data)
 #define ProblemDataSpecificStorage(problem_data)((problem_data)->specific_storage)   //sk
 #define ProblemDataTSlopeX(problem_data)        ((problem_data)->x_slope)   //sk
 #define ProblemDataTSlopeY(problem_data)        ((problem_data)->y_slope)   //sk
+#define ProblemDataChannelWidthX(problem_data)  ((problem_data)->wc_x)
+#define ProblemDataChannelWidthY(problem_data)  ((problem_data)->wc_y)
 #define ProblemDataMannings(problem_data)       ((problem_data)->mann)   //sk
 #define ProblemDataSSlopeX(problem_data)        ((problem_data)->x_sslope)   //RMM
 #define ProblemDataSSlopeY(problem_data)        ((problem_data)->y_sslope)   //RMM
@@ -297,3 +309,4 @@ typedef struct {
 
 
 #endif
+
diff --git a/pfsimulator/parflow_lib/problem_FBx.c b/pfsimulator/parflow_lib/problem_FBx.c
index 672f0ff6c..afd9bdb69 100644
--- a/pfsimulator/parflow_lib/problem_FBx.c
+++ b/pfsimulator/parflow_lib/problem_FBx.c
@@ -1,29 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -217,7 +218,7 @@ PFModule   *FBxNewPublicXtra()
 
       default:
       {
-	InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+        InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
       }
     }
     NA_FreeNameArray(switch_na);
diff --git a/pfsimulator/parflow_lib/problem_FBy.c b/pfsimulator/parflow_lib/problem_FBy.c
index f127fb223..18312dd6c 100644
--- a/pfsimulator/parflow_lib/problem_FBy.c
+++ b/pfsimulator/parflow_lib/problem_FBy.c
@@ -1,29 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -190,7 +191,7 @@ PFModule   *FByNewPublicXtra()
   switch_name = GetStringDefault(name, "False");
   switch_value = NA_NameToIndexExitOnError(switch_na, switch_name, name);
   NA_FreeNameArray(switch_na);
-  
+
   public_xtra->FB_in_y = switch_value;
   if (public_xtra->FB_in_y == 1)
   {
@@ -217,7 +218,7 @@ PFModule   *FByNewPublicXtra()
 
       default:
       {
-	InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+        InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
       }
     }
     NA_FreeNameArray(switch_na);
diff --git a/pfsimulator/parflow_lib/problem_FBz.c b/pfsimulator/parflow_lib/problem_FBz.c
index 443abfd99..3ac6081e2 100644
--- a/pfsimulator/parflow_lib/problem_FBz.c
+++ b/pfsimulator/parflow_lib/problem_FBz.c
@@ -1,29 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -219,11 +220,12 @@ PFModule   *FBzNewPublicXtra()
 
         break;
       }
+
       default:
       {
-	InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+        InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
       }
-    }   
+    }
   }
 
   PFModulePublicXtra(this_module) = public_xtra;
diff --git a/pfsimulator/parflow_lib/problem_bc.c b/pfsimulator/parflow_lib/problem_bc.c
index 33dcfd13a..1647e1921 100644
--- a/pfsimulator/parflow_lib/problem_bc.c
+++ b/pfsimulator/parflow_lib/problem_bc.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routines to help with boundary conditions
diff --git a/pfsimulator/parflow_lib/problem_bc.h b/pfsimulator/parflow_lib/problem_bc.h
index f05fcdda6..a779964a7 100644
--- a/pfsimulator/parflow_lib/problem_bc.h
+++ b/pfsimulator/parflow_lib/problem_bc.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *****************************************************************************/
@@ -76,14 +76,14 @@ typedef struct {
  */
 #define BCStructSubgrids(bc_struct)           ((bc_struct)->subgrids)
 #define BCStructNumSubgrids(bc_struct) \
-  SubgridArrayNumSubgrids(BCStructSubgrids(bc_struct))
+        SubgridArrayNumSubgrids(BCStructSubgrids(bc_struct))
 #define BCStructGrDomain(bc_struct)           ((bc_struct)->gr_domain)
 #define BCStructNumPatches(bc_struct)         ((bc_struct)->num_patches)
 #define BCStructPatchIndexes(bc_struct)       ((bc_struct)->patch_indexes)
 #define BCStructBCTypes(bc_struct)            ((bc_struct)->bc_types)
 #define BCStructValues(bc_struct)             ((bc_struct)->values)
 #define BCStructSubgrid(bc_struct, i) \
-  SubgridArraySubgrid(BCStructSubgrids(bc_struct), i)
+        SubgridArraySubgrid(BCStructSubgrids(bc_struct), i)
 #define BCStructPatchIndex(bc_struct, p)      ((bc_struct)->patch_indexes[p])
 #define BCStructBCType(bc_struct, p)          ((bc_struct)->bc_types[p])
 #define BCStructPatchValues(bc_struct, p, s)  ((bc_struct)->values[p][s])
@@ -96,128 +96,128 @@ typedef struct {
  * @param bc_struct BCStruct struct to read patch count from
  */
 #define ForBCStructNumPatches(ipatch, bc_struct)                    \
-  for(ipatch = 0; ipatch < BCStructNumPatches(bc_struct); ipatch++)
+        for (ipatch = 0; ipatch < BCStructNumPatches(bc_struct); ipatch++)
 
 /*--------------------------------------------------------------------------
  * Looping macro:
  *--------------------------------------------------------------------------*/
 
-#define BCStructPatchLoop(i, j, k, fdir, ival, bc_struct, ipatch, is, body) \
-  {                                                                         \
-    GrGeomSolid  *PV_gr_domain = BCStructGrDomain(bc_struct);               \
-    int PV_patch_index = BCStructPatchIndex(bc_struct, ipatch);             \
-    Subgrid      *PV_subgrid = BCStructSubgrid(bc_struct, is);              \
-                                                                            \
-    int PV_r = SubgridRX(PV_subgrid);                                       \
-    int PV_ix = SubgridIX(PV_subgrid);                                      \
-    int PV_iy = SubgridIY(PV_subgrid);                                      \
-    int PV_iz = SubgridIZ(PV_subgrid);                                      \
-    int PV_nx = SubgridNX(PV_subgrid);                                      \
-    int PV_ny = SubgridNY(PV_subgrid);                                      \
-    int PV_nz = SubgridNZ(PV_subgrid);                                      \
-                                                                            \
-    ival = 0;                                                               \
-    GrGeomPatchLoop(i, j, k, fdir, PV_gr_domain, PV_patch_index,            \
-                    PV_r, PV_ix, PV_iy, PV_iz, PV_nx, PV_ny, PV_nz,         \
-    {                                                                       \
-      body;                                                                 \
-      ival++;                                                               \
-    });                                                                     \
-  }
-
-#define BCStructPatchLoopOvrlnd(i, j, k, fdir, ival, bc_struct, ipatch, is, body) \
-  {                                                                               \
-    GrGeomSolid  *PV_gr_domain = BCStructGrDomain(bc_struct);                     \
-    int PV_patch_index = BCStructPatchIndex(bc_struct, ipatch);                   \
-    Subgrid      *PV_subgrid = BCStructSubgrid(bc_struct, is);                    \
+#define BCStructPatchLoop(i, j, k, fdir, ival, bc_struct, ipatch, is, body)       \
+        {                                                                         \
+          GrGeomSolid  *PV_gr_domain = BCStructGrDomain(bc_struct);               \
+          int PV_patch_index = BCStructPatchIndex(bc_struct, ipatch);             \
+          Subgrid      *PV_subgrid = BCStructSubgrid(bc_struct, is);              \
                                                                                   \
-    int PV_r = SubgridRX(PV_subgrid);                                             \
-    int PV_ix = SubgridIX(PV_subgrid) - 1;                                        \
-    int PV_iy = SubgridIY(PV_subgrid) - 1;                                        \
-    int PV_iz = SubgridIZ(PV_subgrid) - 1;                                        \
-    int PV_nx = SubgridNX(PV_subgrid) + 2;                                        \
-    int PV_ny = SubgridNY(PV_subgrid) + 2;                                        \
-    int PV_nz = SubgridNZ(PV_subgrid) + 2;                                        \
+          int PV_r = SubgridRX(PV_subgrid);                                       \
+          int PV_ix = SubgridIX(PV_subgrid);                                      \
+          int PV_iy = SubgridIY(PV_subgrid);                                      \
+          int PV_iz = SubgridIZ(PV_subgrid);                                      \
+          int PV_nx = SubgridNX(PV_subgrid);                                      \
+          int PV_ny = SubgridNY(PV_subgrid);                                      \
+          int PV_nz = SubgridNZ(PV_subgrid);                                      \
                                                                                   \
-    ival = 0;                                                                     \
-    GrGeomPatchLoop(i, j, k, fdir, PV_gr_domain, PV_patch_index,                  \
-                    PV_r, PV_ix, PV_iy, PV_iz, PV_nx, PV_ny, PV_nz,               \
+          ival = 0;                                                               \
+          GrGeomPatchLoop(i, j, k, fdir, PV_gr_domain, PV_patch_index,            \
+                          PV_r, PV_ix, PV_iy, PV_iz, PV_nx, PV_ny, PV_nz,         \
     {                                                                             \
       body;                                                                       \
       ival++;                                                                     \
     });                                                                           \
-  }
+        }
+
+#define BCStructPatchLoopOvrlnd(i, j, k, fdir, ival, bc_struct, ipatch, is, body)       \
+        {                                                                               \
+          GrGeomSolid  *PV_gr_domain = BCStructGrDomain(bc_struct);                     \
+          int PV_patch_index = BCStructPatchIndex(bc_struct, ipatch);                   \
+          Subgrid      *PV_subgrid = BCStructSubgrid(bc_struct, is);                    \
+                                                                                        \
+          int PV_r = SubgridRX(PV_subgrid);                                             \
+          int PV_ix = SubgridIX(PV_subgrid) - 1;                                        \
+          int PV_iy = SubgridIY(PV_subgrid) - 1;                                        \
+          int PV_iz = SubgridIZ(PV_subgrid) - 1;                                        \
+          int PV_nx = SubgridNX(PV_subgrid) + 2;                                        \
+          int PV_ny = SubgridNY(PV_subgrid) + 2;                                        \
+          int PV_nz = SubgridNZ(PV_subgrid) + 2;                                        \
+                                                                                        \
+          ival = 0;                                                                     \
+          GrGeomPatchLoop(i, j, k, fdir, PV_gr_domain, PV_patch_index,                  \
+                          PV_r, PV_ix, PV_iy, PV_iz, PV_nx, PV_ny, PV_nz,               \
+    {                                                                                   \
+      body;                                                                             \
+      ival++;                                                                           \
+    });                                                                                 \
+        }
 
 /**
  * @brief Variant of BCStructPatchLoop that doesn't utilize an fdir variable, used in ForPatchCells loops.
  *
  * @note Do not call directly! Not intended for use code.
  */
-#define BCStructPatchLoopNoFdir(i, j, k, ival, bc_struct, ipatch, is,       \
-                                locals, setup,                              \
-                                f_left, f_right,                            \
-                                f_down, f_up,                               \
-                                f_back, f_front,                            \
-                                finalize)                                   \
-  {                                                                         \
-    GrGeomSolid  *PV_gr_domain = BCStructGrDomain(bc_struct);               \
-    int PV_patch_index = BCStructPatchIndex(bc_struct, ipatch);             \
-    Subgrid      *PV_subgrid = BCStructSubgrid(bc_struct, is);              \
-                                                                            \
-    int PV_r = SubgridRX(PV_subgrid);                                       \
-    int PV_ix = SubgridIX(PV_subgrid);                                      \
-    int PV_iy = SubgridIY(PV_subgrid);                                      \
-    int PV_iz = SubgridIZ(PV_subgrid);                                      \
-    int PV_nx = SubgridNX(PV_subgrid);                                      \
-    int PV_ny = SubgridNY(PV_subgrid);                                      \
-    int PV_nz = SubgridNZ(PV_subgrid);                                      \
-                                                                            \
-    ival = 0;                                                               \
-    GrGeomPatchLoopNoFdir(i, j, k, PV_gr_domain, PV_patch_index, 0,         \
-                          PV_r, PV_ix, PV_iy,                               \
-                          PV_iz, PV_nx, PV_ny, PV_nz,                       \
-                          locals, setup,                                    \
-                          f_left, f_right,                                  \
-                          f_down, f_up,                                     \
-                          f_back, f_front,                                  \
-    {                                                                       \
-      finalize;                                                             \
-      ival++;                                                               \
-    });                                                                     \
-  }
-
-#define BCStructPatchLoopOvrlndNoFdir(i, j, k, ival, bc_struct, ipatch, is, \
-                                      locals, setup,                        \
-                                      f_left, f_right,                      \
-                                      f_down, f_up,                         \
-                                      f_back, f_front,                      \
-                                      finalize)                             \
-  {                                                                         \
-    GrGeomSolid  *PV_gr_domain = BCStructGrDomain(bc_struct);               \
-    int PV_patch_index = BCStructPatchIndex(bc_struct, ipatch);             \
-    Subgrid      *PV_subgrid = BCStructSubgrid(bc_struct, is);              \
-                                                                            \
-    int PV_r = SubgridRX(PV_subgrid);                                       \
-    int PV_ix = SubgridIX(PV_subgrid) - 1;                                  \
-    int PV_iy = SubgridIY(PV_subgrid) - 1;                                  \
-    int PV_iz = SubgridIZ(PV_subgrid) - 1;                                  \
-    int PV_nx = SubgridNX(PV_subgrid) + 2;                                  \
-    int PV_ny = SubgridNY(PV_subgrid) + 2;                                  \
-    int PV_nz = SubgridNZ(PV_subgrid) + 2;                                  \
-                                                                            \
-    ival = 0;                                                               \
-    GrGeomPatchLoopNoFdir(i, j, k, PV_gr_domain, PV_patch_index, 1,         \
-                          PV_r, PV_ix, PV_iy,                               \
-                          PV_iz, PV_nx, PV_ny, PV_nz,                       \
-                          locals, setup,                                    \
-                          f_left, f_right,                                  \
-                          f_down, f_up,                                     \
-                          f_back, f_front,                                  \
-    {                                                                       \
-      finalize;                                                             \
-      ival++;                                                               \
-    });                                                                     \
-  }
+#define BCStructPatchLoopNoFdir(i, j, k, ival, bc_struct, ipatch, is,             \
+                                locals, setup,                                    \
+                                f_left, f_right,                                  \
+                                f_down, f_up,                                     \
+                                f_back, f_front,                                  \
+                                finalize)                                         \
+        {                                                                         \
+          GrGeomSolid  *PV_gr_domain = BCStructGrDomain(bc_struct);               \
+          int PV_patch_index = BCStructPatchIndex(bc_struct, ipatch);             \
+          Subgrid      *PV_subgrid = BCStructSubgrid(bc_struct, is);              \
+                                                                                  \
+          int PV_r = SubgridRX(PV_subgrid);                                       \
+          int PV_ix = SubgridIX(PV_subgrid);                                      \
+          int PV_iy = SubgridIY(PV_subgrid);                                      \
+          int PV_iz = SubgridIZ(PV_subgrid);                                      \
+          int PV_nx = SubgridNX(PV_subgrid);                                      \
+          int PV_ny = SubgridNY(PV_subgrid);                                      \
+          int PV_nz = SubgridNZ(PV_subgrid);                                      \
+                                                                                  \
+          ival = 0;                                                               \
+          GrGeomPatchLoopNoFdir(i, j, k, PV_gr_domain, PV_patch_index, 0,         \
+                                PV_r, PV_ix, PV_iy,                               \
+                                PV_iz, PV_nx, PV_ny, PV_nz,                       \
+                                locals, setup,                                    \
+                                f_left, f_right,                                  \
+                                f_down, f_up,                                     \
+                                f_back, f_front,                                  \
+    {                                                                             \
+      finalize;                                                                   \
+      ival++;                                                                     \
+    });                                                                           \
+        }
+
+#define BCStructPatchLoopOvrlndNoFdir(i, j, k, ival, bc_struct, ipatch, is,       \
+                                      locals, setup,                              \
+                                      f_left, f_right,                            \
+                                      f_down, f_up,                               \
+                                      f_back, f_front,                            \
+                                      finalize)                                   \
+        {                                                                         \
+          GrGeomSolid  *PV_gr_domain = BCStructGrDomain(bc_struct);               \
+          int PV_patch_index = BCStructPatchIndex(bc_struct, ipatch);             \
+          Subgrid      *PV_subgrid = BCStructSubgrid(bc_struct, is);              \
+                                                                                  \
+          int PV_r = SubgridRX(PV_subgrid);                                       \
+          int PV_ix = SubgridIX(PV_subgrid) - 1;                                  \
+          int PV_iy = SubgridIY(PV_subgrid) - 1;                                  \
+          int PV_iz = SubgridIZ(PV_subgrid) - 1;                                  \
+          int PV_nx = SubgridNX(PV_subgrid) + 2;                                  \
+          int PV_ny = SubgridNY(PV_subgrid) + 2;                                  \
+          int PV_nz = SubgridNZ(PV_subgrid) + 2;                                  \
+                                                                                  \
+          ival = 0;                                                               \
+          GrGeomPatchLoopNoFdir(i, j, k, PV_gr_domain, PV_patch_index, 1,         \
+                                PV_r, PV_ix, PV_iy,                               \
+                                PV_iz, PV_nx, PV_ny, PV_nz,                       \
+                                locals, setup,                                    \
+                                f_left, f_right,                                  \
+                                f_down, f_up,                                     \
+                                f_back, f_front,                                  \
+    {                                                                             \
+      finalize;                                                                   \
+      ival++;                                                                     \
+    });                                                                           \
+        }
 
 /*--------------------------------------------------------------------------
  * ForPatch loops and macros
@@ -288,7 +288,7 @@ typedef struct {
  */
 #define NoLocals ()
 #define UNPACK(locals) _UNPACK locals
-#define _UNPACK(...) __VA_ARGS__ ;
+#define _UNPACK(...) __VA_ARGS__;
 
 /**
  * @brief For use with ForPatchCellsPerFace loop, manages conditional branching internally
@@ -298,23 +298,23 @@ typedef struct {
  * @param[in] fdir Face direction to execute the body on (e.g. FaceLeft)
  * @param[in] body Arbitrary statement body block to execute
  */
-#define FACE(fdir, body)      \
-  case fdir:                  \
-  {                           \
-    body;                     \
-    break;                    \
-  }
+#define FACE(fdir, body)                \
+          case fdir:                    \
+            {                           \
+              body;                     \
+              break;                    \
+            }
 
 /**
  * @brief Used to extract the current BCType from LoopVars in ForPatchCellsPerFace to manage control flow
  */
 #define _GetCurrentPatch(i, j, k, ival, bc_struct, ipatch, is) \
-  BCStructBCType(bc_struct, ipatch)
+        BCStructBCType(bc_struct, ipatch)
 
 #if 0
 /* @MCB: Template for copy+pasting for new BC loops.
- Body blocks intentionally contain ~~ as a statement to cause syntax errors if left alone.
- This is to ensure each cell body statement has been set and not left as a copy+paste.
+ * Body blocks intentionally contain ~~ as a statement to cause syntax errors if left alone.
+ * This is to ensure each cell body statement has been set and not left as a copy+paste.
  */
 ForPatchCellsPerFace(InsertBCTypeHere,
                      BeforeAllCells(DoNothing),
@@ -329,24 +329,24 @@ ForPatchCellsPerFace(InsertBCTypeHere,
                      FACE(FrontFace, { ~~ }),
                      CellFinalize({ ~~ }),
                      AfterAllCells(DoNothing)
-  );
+                     );
 
 /*---------------------------------------------
-  Generic example of a made-up boundary condition using ForPatchCellsPerFace
-  --------------------------------------------- */
+ * Generic example of a made-up boundary condition using ForPatchCellsPerFace
+ * --------------------------------------------- */
 ForPatchCellsPerFace(NotARealBCType,
                      BeforeAllCells({
-                         PFModuleInvokeType(SomeModuleInvoke, some_module, (foo, bar, baz));
-                       }),
+  PFModuleInvokeType(SomeModuleInvoke, some_module, (foo, bar, baz));
+}),
                      LoopVars(i, j, k, ival, bc_struct, ipatch, is),
                      Locals(int im, ip;
                             double some_prod;
-                            double *op;),
+                            double *op; ),
                      CellSetup({
-                         im = SubmatrixEltIndex(J_sub, i, j, k);
-                         ip = SubvectorEltIndex(p_sub, i, j, k);
-                         some_prod = ddp[ip] * dp[ip];
-                       }),
+  im = SubmatrixEltIndex(J_sub, i, j, k);
+  ip = SubvectorEltIndex(p_sub, i, j, k);
+  some_prod = ddp[ip] * dp[ip];
+}),
                      FACE(LeftFace, { op = wp; }),
                      FACE(RightFace, { op = ep; }),
                      FACE(DownFace, { op = sop; }),
@@ -354,9 +354,9 @@ ForPatchCellsPerFace(NotARealBCType,
                      FACE(BackFace, { op = lp; }),
                      FACE(FrontFace, { op = up; }),
                      CellFinalize({
-                         cp[im] += op[im] * some_prod;
-                         op[im] = 0.0;
-                       }),
+  cp[im] += op[im] * some_prod;
+  op[im] = 0.0;
+}),
                      AfterAllCells(DoNothing)
                      );
 
@@ -389,58 +389,58 @@ ForPatchCellsPerFace(NotARealBCType,
  * @param finalize See CellFinalize() macro
  * @param after_loop See AfterAllCells() macro
  */
-#define ForPatchCellsPerFace(bctype,                          \
-                             before_loop,                     \
-                             loopvars, locals,                \
-                             setup,                           \
-                             f_left, f_right,                 \
-                             f_down, f_up,                    \
-                             f_back, f_front,                 \
-                             finalize,                        \
-                             after_loop)                      \
-  {                                                           \
-    if ( ((bctype) == BC_ALL) ||                              \
-         ((bctype) == _GetCurrentPatch(loopvars)))            \
-    {                                                         \
-      before_loop;                                            \
-      BCStructPatchLoopNoFdir(loopvars,                       \
-                              locals, setup,                  \
-                              f_left, f_right,                \
-                              f_down, f_up,                   \
-                              f_back, f_front,                \
-                              finalize);                      \
-      after_loop;                                             \
-    }                                                         \
-  }
+#define ForPatchCellsPerFace(bctype,                                \
+                             before_loop,                           \
+                             loopvars, locals,                      \
+                             setup,                                 \
+                             f_left, f_right,                       \
+                             f_down, f_up,                          \
+                             f_back, f_front,                       \
+                             finalize,                              \
+                             after_loop)                            \
+        {                                                           \
+          if (((bctype) == BC_ALL) ||                               \
+              ((bctype) == _GetCurrentPatch(loopvars)))             \
+          {                                                         \
+            before_loop;                                            \
+            BCStructPatchLoopNoFdir(loopvars,                       \
+                                    locals, setup,                  \
+                                    f_left, f_right,                \
+                                    f_down, f_up,                   \
+                                    f_back, f_front,                \
+                                    finalize);                      \
+            after_loop;                                             \
+          }                                                         \
+        }
 
 /**
  * @brief Variation of ForPatchCellsPerFace() that extends loop bounds to include ghost cells
  * See ForPatchCellsPerFace() for further documentation.  Previously would have been called BCStructPatchLoopOvrlnd.
  */
-#define ForPatchCellsPerFaceWithGhost(bctype, \
-                                      before_loop, loopvars,          \
-                                      locals,                         \
-                                      setup,                          \
-                                      f_left, f_right,                \
-                                      f_down, f_up,                   \
-                                      f_back, f_front,                \
-                                      finalize,                       \
-                                      after_loop)                     \
-  {                                                                   \
-    if ( ((bctype) == BC_ALL) ||                                      \
-         ((bctype) == _GetCurrentPatch(loopvars)))                    \
-    {                                                                 \
-      before_loop;                                                    \
-      BCStructPatchLoopOvrlndNoFdir(loopvars,                         \
-                                    locals,                           \
-                                    setup,                            \
-                                    f_left, f_right,                  \
-                                    f_down, f_up,                     \
-                                    f_back, f_front,                  \
-                                    finalize);                        \
-      after_loop;                                                     \
-    }                                                                 \
-  }
+#define ForPatchCellsPerFaceWithGhost(bctype,                               \
+                                      before_loop, loopvars,                \
+                                      locals,                               \
+                                      setup,                                \
+                                      f_left, f_right,                      \
+                                      f_down, f_up,                         \
+                                      f_back, f_front,                      \
+                                      finalize,                             \
+                                      after_loop)                           \
+        {                                                                   \
+          if (((bctype) == BC_ALL) ||                                       \
+              ((bctype) == _GetCurrentPatch(loopvars)))                     \
+          {                                                                 \
+            before_loop;                                                    \
+            BCStructPatchLoopOvrlndNoFdir(loopvars,                         \
+                                          locals,                           \
+                                          setup,                            \
+                                          f_left, f_right,                  \
+                                          f_down, f_up,                     \
+                                          f_back, f_front,                  \
+                                          finalize);                        \
+            after_loop;                                                     \
+          }                                                                 \
+        }
 
 /**
  * @brief Iterates over cells of a boundary condition patch without any conditional branching
@@ -455,14 +455,14 @@ ForPatchCellsPerFace(NotARealBCType,
  * @param ipatch Current patch number
  * @param is Current subgrid number
  * @param body Statement body to execute
-*/
-#define ForEachPatchCell(i, j, k, ival, bc_struct, ipatch, is, body)    \
-  BCStructPatchLoopNoFdir(i, j, k, ival, bc_struct, ipatch, is,         \
-                          NoLocals,                                     \
-                          body,                                         \
-                          DoNothing, DoNothing,                         \
-                          DoNothing, DoNothing,                         \
-                          DoNothing, DoNothing,                         \
-                          DoNothing);
+ */
+#define ForEachPatchCell(i, j, k, ival, bc_struct, ipatch, is, body)          \
+        BCStructPatchLoopNoFdir(i, j, k, ival, bc_struct, ipatch, is,         \
+                                NoLocals,                                     \
+                                body,                                         \
+                                DoNothing, DoNothing,                         \
+                                DoNothing, DoNothing,                         \
+                                DoNothing, DoNothing,                         \
+                                DoNothing);
 
 #endif
diff --git a/pfsimulator/parflow_lib/problem_bc_internal.c b/pfsimulator/parflow_lib/problem_bc_internal.c
index 76241f0f5..7632b2941 100644
--- a/pfsimulator/parflow_lib/problem_bc_internal.c
+++ b/pfsimulator/parflow_lib/problem_bc_internal.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
diff --git a/pfsimulator/parflow_lib/problem_bc_phase_saturation.c b/pfsimulator/parflow_lib/problem_bc_phase_saturation.c
index 49a5e1288..d507618a0 100644
--- a/pfsimulator/parflow_lib/problem_bc_phase_saturation.c
+++ b/pfsimulator/parflow_lib/problem_bc_phase_saturation.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *****************************************************************************/
@@ -107,7 +107,8 @@ void          BCPhaseSaturation(
   int nx_v, ny_v;
   int sx_v, sy_v, sz_v;
 
-  int indx, ipatch, is, i, j, k, ival=0;
+  int indx, ipatch, is, i, j, k, ival = 0;
+
   PF_UNUSED(ival);
 
   /*-----------------------------------------------------------------------
@@ -152,28 +153,28 @@ void          BCPhaseSaturation(
           double constant;
           ForPatchCellsPerFace(BC_ALL,
                                BeforeAllCells({
-                                   dummy0 = (Type0*)(public_xtra->data[indx + ipatch]);
-                                   constant = (dummy0->constant);
-                                 }),
+            dummy0 = (Type0*)(public_xtra->data[indx + ipatch]);
+            constant = (dummy0->constant);
+          }),
                                LoopVars(i, j, k, ival, bc_struct, ipatch, is),
-                               Locals(int sv, iv;),
+                               Locals(int sv, iv; ),
                                CellSetup({
-                                   sv = 0;
-                                   iv = SubvectorEltIndex(sat_sub, i, j, k);
-                                 }),
-                               FACE(LeftFace,  { sv = -sx_v; }),
+            sv = 0;
+            iv = SubvectorEltIndex(sat_sub, i, j, k);
+          }),
+                               FACE(LeftFace, { sv = -sx_v; }),
                                FACE(RightFace, { sv = sx_v; }),
-                               FACE(DownFace,  { sv = -sy_v; }),
-                               FACE(UpFace,    { sv = sy_v; }),
-                               FACE(BackFace,  { sv = -sz_v; }),
+                               FACE(DownFace, { sv = -sy_v; }),
+                               FACE(UpFace, { sv = sy_v; }),
+                               FACE(BackFace, { sv = -sz_v; }),
                                FACE(FrontFace, { sv = sz_v; }),
                                CellFinalize({
-                                   satp[iv]          = constant;
-                                   satp[iv + sv]     = constant;
-                                   satp[iv + 2 * sv] = constant;
-                                 }),
+            satp[iv] = constant;
+            satp[iv + sv] = constant;
+            satp[iv + 2 * sv] = constant;
+          }),
                                AfterAllCells(DoNothing)
-            );
+                               );
           break;
         }
 
@@ -183,47 +184,47 @@ void          BCPhaseSaturation(
 
           ForPatchCellsPerFace(BC_ALL,
                                BeforeAllCells({
-                                   dummy1 = (Type1*)(public_xtra->data[indx + ipatch]);
-                                   height = (dummy1->height);
-                                   lower = (dummy1->lower);
-                                   upper = (dummy1->upper);
-                                   dz2 = SubgridDZ(subgrid) / 2.0;
-                                 }),
+            dummy1 = (Type1*)(public_xtra->data[indx + ipatch]);
+            height = (dummy1->height);
+            lower = (dummy1->lower);
+            upper = (dummy1->upper);
+            dz2 = SubgridDZ(subgrid) / 2.0;
+          }),
                                LoopVars(i, j, k, ival, bc_struct, ipatch, is),
-                               Locals(int sv, iv; double z;),
+                               Locals(int sv, iv; double z; ),
                                CellSetup({
-                                   sv = 0;
-                                   z = RealSpaceZ(k, SubgridRZ(subgrid));
-                                   iv = SubvectorEltIndex(sat_sub, i, j, k);
-                                 }),
-                               FACE(LeftFace,  { sv = -sx_v; }),
+            sv = 0;
+            z = RealSpaceZ(k, SubgridRZ(subgrid));
+            iv = SubvectorEltIndex(sat_sub, i, j, k);
+          }),
+                               FACE(LeftFace, { sv = -sx_v; }),
                                FACE(RightFace, { sv = sx_v; }),
-                               FACE(DownFace,  { sv = -sy_v; }),
-                               FACE(UpFace,    { sv = sy_v; }),
+                               FACE(DownFace, { sv = -sy_v; }),
+                               FACE(UpFace, { sv = sy_v; }),
                                FACE(BackFace, {
-                                   sv = -sz_v;
-                                   z = z - dz2;
-                                 }),
+            sv = -sz_v;
+            z = z - dz2;
+          }),
                                FACE(FrontFace, {
-                                   sv = sz_v;
-                                   z = z + dz2;
-                                 }),
+            sv = sz_v;
+            z = z + dz2;
+          }),
                                CellFinalize({
-                                   if (z <= height)
-                                   {
-                                     satp[iv       ] = lower;
-                                     satp[iv + sv] = lower;
-                                     satp[iv + 2 * sv] = lower;
-                                   }
-                                   else
-                                   {
-                                     satp[iv       ] = upper;
-                                     satp[iv + sv] = upper;
-                                     satp[iv + 2 * sv] = upper;
-                                   }
-                                 }),
+            if (z <= height)
+            {
+              satp[iv       ] = lower;
+              satp[iv + sv] = lower;
+              satp[iv + 2 * sv] = lower;
+            }
+            else
+            {
+              satp[iv       ] = upper;
+              satp[iv + sv] = upper;
+              satp[iv + 2 * sv] = upper;
+            }
+          }),
                                AfterAllCells(DoNothing)
-            );
+                               );
           break;
         }
 
@@ -240,95 +241,95 @@ void          BCPhaseSaturation(
 
           ForPatchCellsPerFace(BC_ALL,
                                BeforeAllCells({
-                                   dummy2 = (Type2*)(public_xtra->data[indx + ipatch]);
-                                   num_points = (dummy2->num_points);
-                                   point = (dummy2->point);
-                                   height = (dummy2->height);
-                                   lower = (dummy2->lower);
-                                   upper = (dummy2->upper);
-
-                                   dx2 = SubgridDX(subgrid) / 2.0;
-                                   dy2 = SubgridDY(subgrid) / 2.0;
-                                   dz2 = SubgridDZ(subgrid) / 2.0;
-
-                                   /* compute unit direction vector for piecewise linear line */
-                                   unitx = (dummy2->xupper) - (dummy2->xlower);
-                                   unity = (dummy2->yupper) - (dummy2->ylower);
-                                   line_length = sqrt(unitx * unitx + unity * unity);
-                                   unitx /= line_length;
-                                   unity /= line_length;
-                                   line_min = (dummy2->xlower) * unitx
-                                              + (dummy2->ylower) * unity;
-                                 }),
+            dummy2 = (Type2*)(public_xtra->data[indx + ipatch]);
+            num_points = (dummy2->num_points);
+            point = (dummy2->point);
+            height = (dummy2->height);
+            lower = (dummy2->lower);
+            upper = (dummy2->upper);
+
+            dx2 = SubgridDX(subgrid) / 2.0;
+            dy2 = SubgridDY(subgrid) / 2.0;
+            dz2 = SubgridDZ(subgrid) / 2.0;
+
+            /* compute unit direction vector for piecewise linear line */
+            unitx = (dummy2->xupper) - (dummy2->xlower);
+            unity = (dummy2->yupper) - (dummy2->ylower);
+            line_length = sqrt(unitx * unitx + unity * unity);
+            unitx /= line_length;
+            unity /= line_length;
+            line_min = (dummy2->xlower) * unitx
+                       + (dummy2->ylower) * unity;
+          }),
                                LoopVars(i, j, k, ival, bc_struct, ipatch, is),
                                Locals(int sv, iv, ip;
-                                      double x, y, z, xy, slope, interp_height;),
+                                      double x, y, z, xy, slope, interp_height; ),
                                CellSetup({
-                                   sv = 0;
-                                   x = RealSpaceX(i, SubgridRX(subgrid));
-                                   y = RealSpaceY(j, SubgridRY(subgrid));
-                                   z = RealSpaceZ(k, SubgridRZ(subgrid));
-                                   iv = SubvectorEltIndex(sat_sub, i, j, k);
-                                 }),
+            sv = 0;
+            x = RealSpaceX(i, SubgridRX(subgrid));
+            y = RealSpaceY(j, SubgridRY(subgrid));
+            z = RealSpaceZ(k, SubgridRZ(subgrid));
+            iv = SubvectorEltIndex(sat_sub, i, j, k);
+          }),
                                FACE(LeftFace, {
-                                   sv = -sx_v;
-                                   x = x - dx2;
-                                 }),
+            sv = -sx_v;
+            x = x - dx2;
+          }),
                                FACE(RightFace, {
-                                   sv = sx_v;
-                                   x = x + dx2;
-                                 }),
+            sv = sx_v;
+            x = x + dx2;
+          }),
                                FACE(DownFace, {
-                                   sv = -sy_v;
-                                   y = y - dy2;
-                                 }),
+            sv = -sy_v;
+            y = y - dy2;
+          }),
                                FACE(UpFace, {
-                                   sv = sy_v;
-                                   y = y + dy2;
-                                 }),
+            sv = sy_v;
+            y = y + dy2;
+          }),
                                FACE(BackFace, {
-                                   sv = -sz_v;
-                                   z = z - dz2;
-                                 }),
+            sv = -sz_v;
+            z = z - dz2;
+          }),
                                FACE(FrontFace, {
-                                   sv = sz_v;
-                                   z = z + dz2;
-                                 }),
+            sv = sz_v;
+            z = z + dz2;
+          }),
                                CellFinalize(
-                               {
-                                 /* project center of BC face onto piecewise linear line */
-                                 xy = x * unitx + y * unity;
-                                 xy = (xy - line_min) / line_length;
-
-                                 /* find two neighboring points */
-                                 ip = 1;
-                                 for (; ip < (num_points - 1); ip++)
-                                 {
-                                   if (xy < point[ip])
-                                     break;
-                                 }
-
-                                 /* compute the slope */
-                                 slope = ((height[ip] - height[ip - 1]) /
-                                          (point[ip] - point[ip - 1]));
-
-                                 interp_height = height[ip - 1] + slope * (xy - point[ip - 1]);
-
-                                 if (z <= interp_height)
-                                 {
-                                   satp[iv       ] = lower;
-                                   satp[iv + sv] = lower;
-                                   satp[iv + 2 * sv] = lower;
-                                 }
-                                 else
-                                 {
-                                   satp[iv       ] = upper;
-                                   satp[iv + sv] = upper;
-                                   satp[iv + 2 * sv] = upper;
-                                 }
-                               }),
+          {
+            /* project center of BC face onto piecewise linear line */
+            xy = x * unitx + y * unity;
+            xy = (xy - line_min) / line_length;
+
+            /* find two neighboring points */
+            ip = 1;
+            for (; ip < (num_points - 1); ip++)
+            {
+              if (xy < point[ip])
+                break;
+            }
+
+            /* compute the slope */
+            slope = ((height[ip] - height[ip - 1]) /
+                     (point[ip] - point[ip - 1]));
+
+            interp_height = height[ip - 1] + slope * (xy - point[ip - 1]);
+
+            if (z <= interp_height)
+            {
+              satp[iv       ] = lower;
+              satp[iv + sv] = lower;
+              satp[iv + 2 * sv] = lower;
+            }
+            else
+            {
+              satp[iv       ] = upper;
+              satp[iv + sv] = upper;
+              satp[iv + 2 * sv] = upper;
+            }
+          }),
                                AfterAllCells(DoNothing)
-            );
+                               );
 
           break;
         }
@@ -450,11 +451,11 @@ PFModule  *BCPhaseSaturationNewPublicXtra(
 
       public_xtra->patch_indexes[indx] =
         NA_NameToIndex(GlobalsGeometries[domain_index]->patches,
-		       patch_name);
+                       patch_name);
 
-      if(public_xtra->patch_indexes[indx] < 0)
+      if (public_xtra->patch_indexes[indx] < 0)
       {
-	NA_InputError(GlobalsGeometries[domain_index]->patches, patch_name, "");
+        NA_InputError(GlobalsGeometries[domain_index]->patches, patch_name, "");
       }
 
       sprintf(key, "Patch.%s.BCSaturation.%s.Type", patch_name, phase_name);
@@ -546,7 +547,7 @@ PFModule  *BCPhaseSaturationNewPublicXtra(
 
         default:
         {
-	  InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+          InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
         }
       }
       indx++;
diff --git a/pfsimulator/parflow_lib/problem_bc_pressure.c b/pfsimulator/parflow_lib/problem_bc_pressure.c
index 7fd1723d7..371e46ae3 100644
--- a/pfsimulator/parflow_lib/problem_bc_pressure.c
+++ b/pfsimulator/parflow_lib/problem_bc_pressure.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *****************************************************************************/
@@ -214,142 +214,142 @@ BCStruct    *BCPressure(
                                  Locals(int ips, iel, iterations, phase;
                                         double ref_press, ref_den, fcn_val, nonlin_resid;
                                         double z, dtmp, density, density_der;
-                                        double interface_press, offset, height;),
+                                        double interface_press, offset, height; ),
                                  CellSetup({
-                                     ref_press = DirEquilRefPatchValue(interval_data);
-                                     PFModuleInvokeType(PhaseDensityInvoke, phase_density,
-                                                        (0, NULL, NULL, &ref_press, &ref_den,
-                                                         CALCFCN));
-                                     ips = SubvectorEltIndex(z_mult_sub, i, j, k);
-                                     iel = (i - ix) + (j - iy) * nx;
-                                     fcn_val = 0.0;
-                                     nonlin_resid = 1.0;
-                                     iterations = -1;
-                                   }),
-                                 FACE(LeftFace,  { z = rsz_dat[ips]; }),
+              ref_press = DirEquilRefPatchValue(interval_data);
+              PFModuleInvokeType(PhaseDensityInvoke, phase_density,
+                                 (0, NULL, NULL, &ref_press, &ref_den,
+                                  CALCFCN));
+              ips = SubvectorEltIndex(z_mult_sub, i, j, k);
+              iel = (i - ix) + (j - iy) * nx;
+              fcn_val = 0.0;
+              nonlin_resid = 1.0;
+              iterations = -1;
+            }),
+                                 FACE(LeftFace, { z = rsz_dat[ips]; }),
                                  FACE(RightFace, { z = rsz_dat[ips]; }),
-                                 FACE(DownFace,  { z = rsz_dat[ips]; }),
-                                 FACE(UpFace,    { z = rsz_dat[ips]; }),
-                                 FACE(BackFace,  { z = rsz_dat[ips] - dz2 * z_mult_dat[ips]; }),
-                                 FACE(FrontFace, { z = rsz_dat[ips] + dz2 * z_mult_dat[ips];}),
+                                 FACE(DownFace, { z = rsz_dat[ips]; }),
+                                 FACE(UpFace, { z = rsz_dat[ips]; }),
+                                 FACE(BackFace, { z = rsz_dat[ips] - dz2 * z_mult_dat[ips]; }),
+                                 FACE(FrontFace, { z = rsz_dat[ips] + dz2 * z_mult_dat[ips]; }),
                                  CellFinalize(
-                                 {
-                                  /* Solve a nonlinear problem for hydrostatic pressure
-                                   * at points on boundary patch given pressure on reference
-                                   * patch.  Note that the problem is only nonlinear if
-                                   * density depends on pressure.
-                                   *
-                                   * The nonlinear problem to solve is:
-                                   *   F(p) = 0
-                                   *   F(p) = P - P_ref
-                                   *          - 0.5*(rho(P) + rho(P_ref))*gravity*(z - z_ref)
-                                   *
-                                   * Newton's method is used to find a solution. */
-
-                                   while ((nonlin_resid > 1.0E-6) && (iterations < max_its))
-                                   {
-                                     if (iterations > -1)
-                                     {
-                                       PFModuleInvokeType(PhaseDensityInvoke, phase_density,
-                                                          (0, NULL, NULL, &patch_values[ival],
-                                                           &density_der, CALCDER));
-                                       dtmp = 1.0 - 0.5 * density_der * gravity
-                                              * (z - elevations[is][iel]);
-                                       patch_values[ival] = patch_values[ival] - fcn_val / dtmp;
-                                     }
-                                     else
-                                     {
-                                       patch_values[ival] = ref_press;
-                                     }
-                                     PFModuleInvokeType(PhaseDensityInvoke, phase_density,
-                                                        (0, NULL, NULL, &patch_values[ival],
-                                                         &density, CALCFCN));
-
-                                     fcn_val = patch_values[ival] - ref_press
-                                               - 0.5 * (density + ref_den) * gravity
-                                               * (z - elevations[is][iel]);
-                                     nonlin_resid = fabs(fcn_val);
-
-                                     iterations++;
-                                   }            /* End of while loop */
-
-
-                                   /* Iterate over the phases and reset pressures according to
-                                    * hydrostatic conditions with appropriate densities.
-                                    * At each interface, we have hydrostatic conditions, so
-                                    *
-                                    * z_inter = (P_inter - P_ref) /
-                                    *            (0.5*(rho(P_inter)+rho(P_ref))*gravity
-                                    + z_ref
-                                    +
-                                    + Thus, the interface height and pressure are known
-                                    + and hydrostatic conditions can be determined for
-                                    + new phase.
-                                    +
-                                    + NOTE:  This only works for Pc = 0. */
-
-                                   for (phase = 1; phase < num_phases; phase++)
-                                   {
-                                     interface_press = DirEquilRefPatchValueAtInterface(
-                                                                                        interval_data, phase);
-                                     PFModuleInvokeType(PhaseDensityInvoke, phase_density,
-                                                        (phase - 1, NULL, NULL, &interface_press,
-                                                         &interface_den, CALCFCN));
-                                     offset = (interface_press - ref_press)
-                                              / (0.5 * (interface_den + ref_den) * gravity);
-                                     ref_press = interface_press;
-                                     PFModuleInvokeType(PhaseDensityInvoke, phase_density,
-                                                        (phase, NULL, NULL, &ref_press, &ref_den,
-                                                         CALCFCN));
-
-                                     /* Only reset pressure value if in another phase.
-                                      * The following "if" test determines whether this point
-                                      * is in another phase by checking if the computed
-                                      * pressure is less than the interface value.  This
-                                      * test ONLY works if the phases are distributed such
-                                      * that the lighter phases are above the heavier ones. */
-
-                                     if (patch_values[ival] < interface_press)
-                                     {
-                                       height = elevations[is][iel];
-                                       nonlin_resid = 1.0;
-                                       iterations = -1;
-                                       while ((nonlin_resid > 1.0E-6) && (iterations < max_its))
-                                       {
-                                         if (iterations > -1)
-                                         {
-                                           PFModuleInvokeType(PhaseDensityInvoke, phase_density,
-                                                              (phase, NULL, NULL, &patch_values[ival],
-                                                               &density_der, CALCDER));
-
-                                           dtmp = 1.0 - 0.5 * density_der * gravity
-                                                  * (z - height);
-                                           patch_values[ival] = patch_values[ival]
-                                                                - fcn_val / dtmp;
-                                         }
-                                         else
-                                         {
-                                           height = height + offset;
-                                           patch_values[ival] = ref_press;
-                                         }
-
-                                         PFModuleInvokeType(PhaseDensityInvoke, phase_density,
-                                                            (phase, NULL, NULL,
-                                                             &patch_values[ival], &density,
-                                                             CALCFCN));
-
-                                         fcn_val = patch_values[ival] - ref_press
-                                                   - 0.5 * (density + ref_den)
-                                                   * gravity * (z - height);
-                                         nonlin_resid = fabs(fcn_val);
-
-                                         iterations++;
-                                       }              /* End of while loop */
-                                     }                /* End if above interface */
-                                   }                  /* End phase loop */
-                                 }),
+            {
+              /* Solve a nonlinear problem for hydrostatic pressure
+               * at points on boundary patch given pressure on reference
+               * patch.  Note that the problem is only nonlinear if
+               * density depends on pressure.
+               *
+               * The nonlinear problem to solve is:
+               *   F(p) = 0
+               *   F(p) = P - P_ref
+               *          - 0.5*(rho(P) + rho(P_ref))*gravity*(z - z_ref)
+               *
+               * Newton's method is used to find a solution. */
+
+              while ((nonlin_resid > 1.0E-6) && (iterations < max_its))
+              {
+                if (iterations > -1)
+                {
+                  PFModuleInvokeType(PhaseDensityInvoke, phase_density,
+                                     (0, NULL, NULL, &patch_values[ival],
+                                      &density_der, CALCDER));
+                  dtmp = 1.0 - 0.5 * density_der * gravity
+                         * (z - elevations[is][iel]);
+                  patch_values[ival] = patch_values[ival] - fcn_val / dtmp;
+                }
+                else
+                {
+                  patch_values[ival] = ref_press;
+                }
+                PFModuleInvokeType(PhaseDensityInvoke, phase_density,
+                                   (0, NULL, NULL, &patch_values[ival],
+                                    &density, CALCFCN));
+
+                fcn_val = patch_values[ival] - ref_press
+                          - 0.5 * (density + ref_den) * gravity
+                          * (z - elevations[is][iel]);
+                nonlin_resid = fabs(fcn_val);
+
+                iterations++;
+              }                                 /* End of while loop */
+
+
+              /* Iterate over the phases and reset pressures according to
+               * hydrostatic conditions with appropriate densities.
+               * At each interface, we have hydrostatic conditions, so
+               *
+               * z_inter = (P_inter - P_ref) /
+               *            (0.5*(rho(P_inter)+rho(P_ref))*gravity
+               + z_ref
+               +
+               + Thus, the interface height and pressure are known
+               + and hydrostatic conditions can be determined for
+               + new phase.
+               +
+               + NOTE:  This only works for Pc = 0. */
+
+              for (phase = 1; phase < num_phases; phase++)
+              {
+                interface_press = DirEquilRefPatchValueAtInterface(
+                                                                   interval_data, phase);
+                PFModuleInvokeType(PhaseDensityInvoke, phase_density,
+                                   (phase - 1, NULL, NULL, &interface_press,
+                                    &interface_den, CALCFCN));
+                offset = (interface_press - ref_press)
+                         / (0.5 * (interface_den + ref_den) * gravity);
+                ref_press = interface_press;
+                PFModuleInvokeType(PhaseDensityInvoke, phase_density,
+                                   (phase, NULL, NULL, &ref_press, &ref_den,
+                                    CALCFCN));
+
+                /* Only reset pressure value if in another phase.
+                 * The following "if" test determines whether this point
+                 * is in another phase by checking if the computed
+                 * pressure is less than the interface value.  This
+                 * test ONLY works if the phases are distributed such
+                 * that the lighter phases are above the heavier ones. */
+
+                if (patch_values[ival] < interface_press)
+                {
+                  height = elevations[is][iel];
+                  nonlin_resid = 1.0;
+                  iterations = -1;
+                  while ((nonlin_resid > 1.0E-6) && (iterations < max_its))
+                  {
+                    if (iterations > -1)
+                    {
+                      PFModuleInvokeType(PhaseDensityInvoke, phase_density,
+                                         (phase, NULL, NULL, &patch_values[ival],
+                                          &density_der, CALCDER));
+
+                      dtmp = 1.0 - 0.5 * density_der * gravity
+                             * (z - height);
+                      patch_values[ival] = patch_values[ival]
+                                           - fcn_val / dtmp;
+                    }
+                    else
+                    {
+                      height = height + offset;
+                      patch_values[ival] = ref_press;
+                    }
+
+                    PFModuleInvokeType(PhaseDensityInvoke, phase_density,
+                                       (phase, NULL, NULL,
+                                        &patch_values[ival], &density,
+                                        CALCFCN));
+
+                    fcn_val = patch_values[ival] - ref_press
+                              - 0.5 * (density + ref_den)
+                              * gravity * (z - height);
+                    nonlin_resid = fabs(fcn_val);
+
+                    iterations++;
+                  }                                   /* End of while loop */
+                }                                     /* End if above interface */
+              }                                       /* End phase loop */
+            }),
                                  AfterAllCells(DoNothing)
-              ); /* End ForPatchCellsPerFace loop */
+                                 ); /* End ForPatchCellsPerFace loop */
           }                      /* End subgrid loop */
 
 
@@ -421,160 +421,160 @@ BCStruct    *BCPressure(
                                  BeforeAllCells(DoNothing),
                                  LoopVars(i, j, k, ival, bc_struct, ipatch, is),
                                  Locals(int ips, iterations;
-                                        double x, y, z, fcn_val, nonlin_resid;),
+                                        double x, y, z, fcn_val, nonlin_resid; ),
                                  CellSetup({
-                                     ips = SubvectorEltIndex(z_mult_sub, i, j, k);
-                                     x = RealSpaceX(i, SubgridRX(subgrid));
-                                     y = RealSpaceY(j, SubgridRY(subgrid));
-                                     z = rsz_dat[ips];
-                                     fcn_val = 0.0;
-                                     nonlin_resid = 1.0;
-                                     iterations = -1;
-                                   }),
-                                 FACE(LeftFace,  { x = x - dx2; }),
+              ips = SubvectorEltIndex(z_mult_sub, i, j, k);
+              x = RealSpaceX(i, SubgridRX(subgrid));
+              y = RealSpaceY(j, SubgridRY(subgrid));
+              z = rsz_dat[ips];
+              fcn_val = 0.0;
+              nonlin_resid = 1.0;
+              iterations = -1;
+            }),
+                                 FACE(LeftFace, { x = x - dx2; }),
                                  FACE(RightFace, { x = x + dx2; }),
-                                 FACE(DownFace,  { y = y - dy2; }),
-                                 FACE(UpFace,    { y = y + dy2; }),
-                                 FACE(BackFace,  { z = z - dz2 * z_mult_dat[ips]; }),
+                                 FACE(DownFace, { y = y - dy2; }),
+                                 FACE(UpFace, { y = y + dy2; }),
+                                 FACE(BackFace, { z = z - dz2 * z_mult_dat[ips]; }),
                                  FACE(FrontFace, { z = z + dz2 * z_mult_dat[ips]; }),
                                  CellFinalize(
-                                 {
-                                   /* project center of BC face onto piecewise line */
-                                   xy = (x * unitx + y * unity - line_min) / line_length;
-
-                                   /* find two neighboring points */
-                                   ip = 1;
-                                   num_points = DirEquilPLinearNumPoints(interval_data);
-                                   for (; ip < (num_points - 1); ip++)
-                                   {
-                                     if (xy < DirEquilPLinearPoint(interval_data, ip))
-                                       break;
-                                   }
-
-                                   /* compute the slope */
-                                   slope = ((DirEquilPLinearValue(interval_data, ip)
-                                             - DirEquilPLinearValue(interval_data, (ip - 1)))
-                                            / (DirEquilPLinearPoint(interval_data, ip)
-                                               - DirEquilPLinearPoint(interval_data, (ip - 1))));
-
-                                   ref_press = DirEquilPLinearValue(interval_data, ip - 1)
-                                               + slope * (xy - DirEquilPLinearPoint(interval_data, ip - 1));
-                                   PFModuleInvokeType(PhaseDensityInvoke, phase_density,
-                                                      (0, NULL, NULL, &ref_press, &ref_den,
-                                                       CALCFCN));
-
-                                   /* Solve a nonlinear problem for hydrostatic pressure
-                                    * at points on boundary patch given reference pressure.
-                                    * Note that the problem is only nonlinear if
-                                    * density depends on pressure.
-                                    *
-                                    * The nonlinear problem to solve is:
-                                    *   F(p) = 0
-                                    *   F(p) = P - P_ref
-                                    *          - 0.5*(rho(P) + rho(P_ref))*gravity*z
-                                    *
-                                    * Newton's method is used to find a solution. */
-
-                                   while ((nonlin_resid > 1.0E-6) && (iterations < max_its))
-                                   {
-                                     if (iterations > -1)
-                                     {
-                                       PFModuleInvokeType(PhaseDensityInvoke, phase_density,
-                                                          (0, NULL, NULL, &patch_values[ival],
-                                                           &density_der, CALCDER));
-                                       dtmp = 1.0 - 0.5 * density_der * gravity * z;
-                                       patch_values[ival] = patch_values[ival] - fcn_val / dtmp;
-                                     }
-                                     else
-                                     {
-                                       patch_values[ival] = ref_press;
-                                     }
-                                     PFModuleInvokeType(PhaseDensityInvoke, phase_density,
-                                                        (0, NULL, NULL, &patch_values[ival],
-                                                         &density, CALCFCN));
-
-                                     fcn_val = patch_values[ival] - ref_press
-                                               - 0.5 * (density + ref_den) * gravity * z;
-                                     nonlin_resid = fabs(fcn_val);
-
-                                     iterations++;
-                                   }            /* End of while loop */
-
-                                   /* Iterate over the phases and reset pressures according to
-                                    * hydrostatic conditions with appropriate densities.
-                                    * At each interface, we have hydrostatic conditions, so
-                                    *
-                                    * z_inter = (P_inter - P_ref) /
-                                    *            (0.5*(rho(P_inter)+rho(P_ref))*gravity
-                                    + z_ref
-                                    +
-                                    + Thus, the interface height and pressure are known
-                                    + and hydrostatic conditions can be determined for
-                                    + new phase.
-                                    +
-                                    + NOTE:  This only works for Pc = 0. */
-
-                                   for (phase = 1; phase < num_phases; phase++)
-                                   {
-                                     interface_press = DirEquilPLinearValueAtInterface(
-                                                                                       interval_data, phase);
-                                     PFModuleInvokeType(PhaseDensityInvoke, phase_density,
-                                                        (phase - 1, NULL, NULL, &interface_press,
-                                                         &interface_den, CALCFCN));
-                                     offset = (interface_press - ref_press)
-                                              / (0.5 * (interface_den + ref_den) * gravity);
-                                     ref_press = interface_press;
-                                     PFModuleInvokeType(PhaseDensityInvoke, phase_density,
-                                                        (phase, NULL, NULL, &ref_press, &ref_den,
-                                                         CALCFCN));
-
-                                     /* Only reset pressure value if in another phase.
-                                      * The following "if" test determines whether this point
-                                      * is in another phase by checking if the computed
-                                      * pressure is less than the interface value.  This
-                                      * test ONLY works if the phases are distributed such
-                                      * that the lighter phases are above the heavier ones. */
-
-                                     if (patch_values[ival] < interface_press)
-                                     {
-                                       height = 0.0;
-                                       nonlin_resid = 1.0;
-                                       iterations = -1;
-                                       while ((nonlin_resid > 1.0E-6) && (iterations < max_its))
-                                       {
-                                         if (iterations > -1)
-                                         {
-                                           PFModuleInvokeType(PhaseDensityInvoke, phase_density,
-                                                              (phase, NULL, NULL, &patch_values[ival],
-                                                               &density_der, CALCDER));
-
-                                           dtmp = 1.0 - 0.5 * density_der * gravity * (z - height);
-                                           patch_values[ival] = patch_values[ival]
-                                                                - fcn_val / dtmp;
-                                         }
-                                         else
-                                         {
-                                           height = height + offset;
-                                           patch_values[ival] = ref_press;
-                                         }
-
-                                         PFModuleInvokeType(PhaseDensityInvoke, phase_density,
-                                                            (phase, NULL, NULL,
-                                                             &patch_values[ival], &density,
-                                                             CALCFCN));
-
-                                         fcn_val = patch_values[ival] - ref_press
-                                                   - 0.5 * (density + ref_den) * gravity
-                                                   * (z - height);
-                                         nonlin_resid = fabs(fcn_val);
-
-                                         iterations++;
-                                       }              /* End of while loop */
-                                     }                /* End if above interface */
-                                   }                  /* End phase loop */
-                                 }),
+            {
+              /* project center of BC face onto piecewise line */
+              xy = (x * unitx + y * unity - line_min) / line_length;
+
+              /* find two neighboring points */
+              ip = 1;
+              num_points = DirEquilPLinearNumPoints(interval_data);
+              for (; ip < (num_points - 1); ip++)
+              {
+                if (xy < DirEquilPLinearPoint(interval_data, ip))
+                  break;
+              }
+
+              /* compute the slope */
+              slope = ((DirEquilPLinearValue(interval_data, ip)
+                        - DirEquilPLinearValue(interval_data, (ip - 1)))
+                       / (DirEquilPLinearPoint(interval_data, ip)
+                          - DirEquilPLinearPoint(interval_data, (ip - 1))));
+
+              ref_press = DirEquilPLinearValue(interval_data, ip - 1)
+                          + slope * (xy - DirEquilPLinearPoint(interval_data, ip - 1));
+              PFModuleInvokeType(PhaseDensityInvoke, phase_density,
+                                 (0, NULL, NULL, &ref_press, &ref_den,
+                                  CALCFCN));
+
+              /* Solve a nonlinear problem for hydrostatic pressure
+               * at points on boundary patch given reference pressure.
+               * Note that the problem is only nonlinear if
+               * density depends on pressure.
+               *
+               * The nonlinear problem to solve is:
+               *   F(p) = 0
+               *   F(p) = P - P_ref
+               *          - 0.5*(rho(P) + rho(P_ref))*gravity*z
+               *
+               * Newton's method is used to find a solution. */
+
+              while ((nonlin_resid > 1.0E-6) && (iterations < max_its))
+              {
+                if (iterations > -1)
+                {
+                  PFModuleInvokeType(PhaseDensityInvoke, phase_density,
+                                     (0, NULL, NULL, &patch_values[ival],
+                                      &density_der, CALCDER));
+                  dtmp = 1.0 - 0.5 * density_der * gravity * z;
+                  patch_values[ival] = patch_values[ival] - fcn_val / dtmp;
+                }
+                else
+                {
+                  patch_values[ival] = ref_press;
+                }
+                PFModuleInvokeType(PhaseDensityInvoke, phase_density,
+                                   (0, NULL, NULL, &patch_values[ival],
+                                    &density, CALCFCN));
+
+                fcn_val = patch_values[ival] - ref_press
+                          - 0.5 * (density + ref_den) * gravity * z;
+                nonlin_resid = fabs(fcn_val);
+
+                iterations++;
+              }                                 /* End of while loop */
+
+              /* Iterate over the phases and reset pressures according to
+               * hydrostatic conditions with appropriate densities.
+               * At each interface, we have hydrostatic conditions, so
+               *
+               * z_inter = (P_inter - P_ref) /
+               *            (0.5*(rho(P_inter)+rho(P_ref))*gravity
+               + z_ref
+               +
+               + Thus, the interface height and pressure are known
+               + and hydrostatic conditions can be determined for
+               + new phase.
+               +
+               + NOTE:  This only works for Pc = 0. */
+
+              for (phase = 1; phase < num_phases; phase++)
+              {
+                interface_press = DirEquilPLinearValueAtInterface(
+                                                                  interval_data, phase);
+                PFModuleInvokeType(PhaseDensityInvoke, phase_density,
+                                   (phase - 1, NULL, NULL, &interface_press,
+                                    &interface_den, CALCFCN));
+                offset = (interface_press - ref_press)
+                         / (0.5 * (interface_den + ref_den) * gravity);
+                ref_press = interface_press;
+                PFModuleInvokeType(PhaseDensityInvoke, phase_density,
+                                   (phase, NULL, NULL, &ref_press, &ref_den,
+                                    CALCFCN));
+
+                /* Only reset pressure value if in another phase.
+                 * The following "if" test determines whether this point
+                 * is in another phase by checking if the computed
+                 * pressure is less than the interface value.  This
+                 * test ONLY works if the phases are distributed such
+                 * that the lighter phases are above the heavier ones. */
+
+                if (patch_values[ival] < interface_press)
+                {
+                  height = 0.0;
+                  nonlin_resid = 1.0;
+                  iterations = -1;
+                  while ((nonlin_resid > 1.0E-6) && (iterations < max_its))
+                  {
+                    if (iterations > -1)
+                    {
+                      PFModuleInvokeType(PhaseDensityInvoke, phase_density,
+                                         (phase, NULL, NULL, &patch_values[ival],
+                                          &density_der, CALCDER));
+
+                      dtmp = 1.0 - 0.5 * density_der * gravity * (z - height);
+                      patch_values[ival] = patch_values[ival]
+                                           - fcn_val / dtmp;
+                    }
+                    else
+                    {
+                      height = height + offset;
+                      patch_values[ival] = ref_press;
+                    }
+
+                    PFModuleInvokeType(PhaseDensityInvoke, phase_density,
+                                       (phase, NULL, NULL,
+                                        &patch_values[ival], &density,
+                                        CALCFCN));
+
+                    fcn_val = patch_values[ival] - ref_press
+                              - 0.5 * (density + ref_den) * gravity
+                              * (z - height);
+                    nonlin_resid = fabs(fcn_val);
+
+                    iterations++;
+                  }                                   /* End of while loop */
+                }                                     /* End if above interface */
+              }                                       /* End phase loop */
+            }),
                                  AfterAllCells(DoNothing)
-              ); /* End ForPatchCellsPerFace */
+                                 ); /* End ForPatchCellsPerFace */
           }
           break;
         } /* End DirEquilPLinear */
@@ -637,20 +637,20 @@ BCStruct    *BCPressure(
             ForPatchCellsPerFace(BC_ALL,
                                  BeforeAllCells(DoNothing),
                                  LoopVars(i, j, k, ival, bc_struct, ipatch, is),
-                                 Locals(int ips;),
+                                 Locals(int ips; ),
                                  CellSetup({
-                                     patch_values_size++;
-                                     ips = SubvectorEltIndex(z_mult_sub, i, j, k);
-                                   }),
-                                 FACE(LeftFace,  { area += dy * dz * z_mult_dat[ips]; }),
+              patch_values_size++;
+              ips = SubvectorEltIndex(z_mult_sub, i, j, k);
+            }),
+                                 FACE(LeftFace, { area += dy * dz * z_mult_dat[ips]; }),
                                  FACE(RightFace, { area += dy * dz * z_mult_dat[ips]; }),
-                                 FACE(DownFace,  { area += dx * dz * z_mult_dat[ips]; }),
-                                 FACE(UpFace,    { area += dx * dz * z_mult_dat[ips]; }),
-                                 FACE(BackFace,  { area += dx * dy; }),
+                                 FACE(DownFace, { area += dx * dz * z_mult_dat[ips]; }),
+                                 FACE(UpFace, { area += dx * dz * z_mult_dat[ips]; }),
+                                 FACE(BackFace, { area += dx * dy; }),
                                  FACE(FrontFace, { area += dx * dy; }),
                                  CellFinalize(DoNothing),
                                  AfterAllCells(DoNothing)
-              );
+                                 );
 
             patch_values = talloc(double, patch_values_size);
             memset(patch_values, 0, patch_values_size * sizeof(double));
@@ -719,7 +719,7 @@ BCStruct    *BCPressure(
             tmpp = SubvectorData(subvector);
             ForEachPatchCell(i, j, k, ival, bc_struct, ipatch, is,
             {
-				/*int ips = SubvectorEltIndex(z_mult_sub, i, j, k);*/
+              /*int ips = SubvectorEltIndex(z_mult_sub, i, j, k);*/
               itmp = SubvectorEltIndex(subvector, i, j, k);
 
               patch_values[ival] = tmpp[itmp];     /*- density*gravity*z;*/
@@ -819,15 +819,15 @@ BCStruct    *BCPressure(
                 ForPatchCellsPerFace(BC_ALL,
                                      BeforeAllCells(DoNothing),
                                      LoopVars(i, j, k, ival, bc_struct, ipatch, is),
-                                     Locals(double x;),
-                                     CellSetup(  { x = RealSpaceX(i, SubgridRX(subgrid)); }),
-                                     FACE(LeftFace,  { x = x - dx2; }),
+                                     Locals(double x; ),
+                                     CellSetup({ x = RealSpaceX(i, SubgridRX(subgrid)); }),
+                                     FACE(LeftFace, { x = x - dx2; }),
                                      FACE(RightFace, { x = x + dx2; }),
                                      FACE(DownFace, DoNothing), FACE(UpFace, DoNothing),
                                      FACE(BackFace, DoNothing), FACE(FrontFace, DoNothing),
                                      CellFinalize({ patch_values[ival] = x; }),
                                      AfterAllCells(DoNothing)
-                  );
+                                     );
                 break;
               }     /* End case 1 */
 
@@ -837,22 +837,22 @@ BCStruct    *BCPressure(
                                      BeforeAllCells(DoNothing),
                                      LoopVars(i, j, k, ival, bc_struct, ipatch, is),
                                      Locals(int ips;
-                                            double x, y, z;),
+                                            double x, y, z; ),
                                      CellSetup({
-                                         ips = SubvectorEltIndex(z_mult_sub, i, j, k);
-                                         x = RealSpaceX(i, SubgridRX(subgrid));
-                                         y = RealSpaceY(j, SubgridRY(subgrid));
-                                         z = rsz_dat[ips];
-                                       }),
-                                     FACE(LeftFace,    { x = x - dx2; }),
-                                     FACE(RightFace,   { x = x + dx2; }),
-                                     FACE(DownFace,    { y = y - dy2; }),
-                                     FACE(UpFace,      { y = y + dy2; }),
-                                     FACE(BackFace,    { z = z - dz2 * z_mult_dat[ips]; }),
-                                     FACE(FrontFace,   { z = z + dz2 * z_mult_dat[ips]; }),
+                  ips = SubvectorEltIndex(z_mult_sub, i, j, k);
+                  x = RealSpaceX(i, SubgridRX(subgrid));
+                  y = RealSpaceY(j, SubgridRY(subgrid));
+                  z = rsz_dat[ips];
+                }),
+                                     FACE(LeftFace, { x = x - dx2; }),
+                                     FACE(RightFace, { x = x + dx2; }),
+                                     FACE(DownFace, { y = y - dy2; }),
+                                     FACE(UpFace, { y = y + dy2; }),
+                                     FACE(BackFace, { z = z - dz2 * z_mult_dat[ips]; }),
+                                     FACE(FrontFace, { z = z + dz2 * z_mult_dat[ips]; }),
                                      CellFinalize({ patch_values[ival] = x + y + z; }),
                                      AfterAllCells(DoNothing)
-                  );
+                                     );
                 break;
               }     /* End case 2 */
 
@@ -861,21 +861,21 @@ BCStruct    *BCPressure(
                 ForPatchCellsPerFace(BC_ALL,
                                      BeforeAllCells(DoNothing),
                                      LoopVars(i, j, k, ival, bc_struct, ipatch, is),
-                                     Locals(double x, y;),
+                                     Locals(double x, y; ),
                                      CellSetup({
-                                         x = RealSpaceX(i, SubgridRX(subgrid));
-                                         y = RealSpaceY(j, SubgridRY(subgrid));
-                                       }),
-                                     FACE(LeftFace,  { x = x - dx2; }),
+                  x = RealSpaceX(i, SubgridRX(subgrid));
+                  y = RealSpaceY(j, SubgridRY(subgrid));
+                }),
+                                     FACE(LeftFace, { x = x - dx2; }),
                                      FACE(RightFace, { x = x + dx2; }),
-                                     FACE(DownFace,  { y = y - dy2; }),
-                                     FACE(UpFace,    { y = y + dy2; }),
+                                     FACE(DownFace, { y = y - dy2; }),
+                                     FACE(UpFace, { y = y + dy2; }),
                                      FACE(BackFace, DoNothing), FACE(FrontFace, DoNothing),
                                      CellFinalize({
-                                         patch_values[ival] = x * x * x * y * y + sin(x * y) + 1;
-                                       }),
+                  patch_values[ival] = x * x * x * y * y + sin(x * y) + 1;
+                }),
                                      AfterAllCells(DoNothing)
-                  );
+                                     );
                 break;
               }     /* End case 3 */
 
@@ -884,21 +884,21 @@ BCStruct    *BCPressure(
                 ForPatchCellsPerFace(BC_ALL,
                                      BeforeAllCells(DoNothing),
                                      LoopVars(i, j, k, ival, bc_struct, ipatch, is),
-                                     Locals(double x, y;),
+                                     Locals(double x, y; ),
                                      CellSetup({
-                                         x = RealSpaceX(i, SubgridRX(subgrid));
-                                         y = RealSpaceY(j, SubgridRY(subgrid));
-                                       }),
-                                     FACE(LeftFace,  { x = x - dx2; }),
+                  x = RealSpaceX(i, SubgridRX(subgrid));
+                  y = RealSpaceY(j, SubgridRY(subgrid));
+                }),
+                                     FACE(LeftFace, { x = x - dx2; }),
                                      FACE(RightFace, { x = x + dx2; }),
-                                     FACE(DownFace,  { y = y - dy2; }),
-                                     FACE(UpFace,    { y = y + dy2; }),
+                                     FACE(DownFace, { y = y - dy2; }),
+                                     FACE(UpFace, { y = y + dy2; }),
                                      FACE(BackFace, DoNothing), FACE(FrontFace, DoNothing),
                                      CellFinalize({
-                                         patch_values[ival] = pow(x, 3) * pow(y, 4) + x * x + sin(x * y) * cos(y) + 1;
-                                       }),
+                  patch_values[ival] = pow(x, 3) * pow(y, 4) + x * x + sin(x * y) * cos(y) + 1;
+                }),
                                      AfterAllCells(DoNothing)
-                  );
+                                     );
                 break;
               }     /* End case 4 */
 
@@ -907,22 +907,22 @@ BCStruct    *BCPressure(
                 ForPatchCellsPerFace(BC_ALL,
                                      BeforeAllCells(DoNothing),
                                      LoopVars(i, j, k, ival, bc_struct, ipatch, is),
-                                     Locals(int ips; double x, y, z;),
+                                     Locals(int ips; double x, y, z; ),
                                      CellSetup({
-                                         ips = SubvectorEltIndex(z_mult_sub, i, j, k);
-                                         x = RealSpaceX(i, SubgridRX(subgrid));
-                                         y = RealSpaceY(j, SubgridRY(subgrid));
-                                         z = rsz_dat[ips];
-                                       }),
-                                     FACE(LeftFace,  { x = x - dx2; }),
+                  ips = SubvectorEltIndex(z_mult_sub, i, j, k);
+                  x = RealSpaceX(i, SubgridRX(subgrid));
+                  y = RealSpaceY(j, SubgridRY(subgrid));
+                  z = rsz_dat[ips];
+                }),
+                                     FACE(LeftFace, { x = x - dx2; }),
                                      FACE(RightFace, { x = x + dx2; }),
-                                     FACE(DownFace,  { y = y - dy2; }),
-                                     FACE(UpFace,    { y = y + dy2; }),
-                                     FACE(BackFace,  { z = z - dz2 * z_mult_dat[ips]; }),
+                                     FACE(DownFace, { y = y - dy2; }),
+                                     FACE(UpFace, { y = y + dy2; }),
+                                     FACE(BackFace, { z = z - dz2 * z_mult_dat[ips]; }),
                                      FACE(FrontFace, { z = z + dz2 * z_mult_dat[ips]; }),
                                      CellFinalize({ patch_values[ival] = x * y * z * time + 1; }),
                                      AfterAllCells(DoNothing)
-                  );
+                                     );
                 break;
               }     /* End case 5 */
 
@@ -931,22 +931,22 @@ BCStruct    *BCPressure(
                 ForPatchCellsPerFace(BC_ALL,
                                      BeforeAllCells(DoNothing),
                                      LoopVars(i, j, k, ival, bc_struct, ipatch, is),
-                                     Locals(int ips; double x, y, z;),
+                                     Locals(int ips; double x, y, z; ),
                                      CellSetup({
-                                         ips = SubvectorEltIndex(z_mult_sub, i, j, k);
-                                         x = RealSpaceX(i, SubgridRX(subgrid));
-                                         y = RealSpaceY(j, SubgridRY(subgrid));
-                                         z = rsz_dat[ips];
-                                       }),
-                                     FACE(LeftFace,  { x = x - dx2; }),
+                  ips = SubvectorEltIndex(z_mult_sub, i, j, k);
+                  x = RealSpaceX(i, SubgridRX(subgrid));
+                  y = RealSpaceY(j, SubgridRY(subgrid));
+                  z = rsz_dat[ips];
+                }),
+                                     FACE(LeftFace, { x = x - dx2; }),
                                      FACE(RightFace, { x = x + dx2; }),
-                                     FACE(DownFace,  { y = y - dy2; }),
-                                     FACE(UpFace,    { y = y + dy2; }),
-                                     FACE(BackFace,  { z = z - dz2 * z_mult_dat[ips]; }),
+                                     FACE(DownFace, { y = y - dy2; }),
+                                     FACE(UpFace, { y = y + dy2; }),
+                                     FACE(BackFace, { z = z - dz2 * z_mult_dat[ips]; }),
                                      FACE(FrontFace, { z = z + dz2 * z_mult_dat[ips]; }),
                                      CellFinalize({ patch_values[ival] = x * y * z * time + 1; }),
                                      AfterAllCells(DoNothing)
-                  );
+                                     );
                 break;
               }     /* End case 5 */
             }       /* End switch */
diff --git a/pfsimulator/parflow_lib/problem_capillary_pressure.c b/pfsimulator/parflow_lib/problem_capillary_pressure.c
index bee5e1001..3f880dc0d 100644
--- a/pfsimulator/parflow_lib/problem_capillary_pressure.c
+++ b/pfsimulator/parflow_lib/problem_capillary_pressure.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -264,7 +264,7 @@ PFModule  *CapillaryPressureNewPublicXtra(
 
       default:
       {
-	InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+        InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
       }
     }
   }
diff --git a/pfsimulator/parflow_lib/problem_domain.c b/pfsimulator/parflow_lib/problem_domain.c
index 7b1996cbf..2bc3160ec 100644
--- a/pfsimulator/parflow_lib/problem_domain.c
+++ b/pfsimulator/parflow_lib/problem_domain.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
diff --git a/pfsimulator/parflow_lib/problem_dz_scale.c b/pfsimulator/parflow_lib/problem_dz_scale.c
index 2c97cc841..7be80fb61 100644
--- a/pfsimulator/parflow_lib/problem_dz_scale.c
+++ b/pfsimulator/parflow_lib/problem_dz_scale.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
diff --git a/pfsimulator/parflow_lib/problem_eval.c b/pfsimulator/parflow_lib/problem_eval.c
index 990c25a37..4f9a3976c 100644
--- a/pfsimulator/parflow_lib/problem_eval.c
+++ b/pfsimulator/parflow_lib/problem_eval.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routine to help generate evaluation points (w.r.t. f' and h')
diff --git a/pfsimulator/parflow_lib/problem_eval.h b/pfsimulator/parflow_lib/problem_eval.h
index fb7c8d943..a014c3305 100644
--- a/pfsimulator/parflow_lib/problem_eval.h
+++ b/pfsimulator/parflow_lib/problem_eval.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Header info for evaluation points,
 * used with root-finding/maximization procedures.
diff --git a/pfsimulator/parflow_lib/problem_geometries.c b/pfsimulator/parflow_lib/problem_geometries.c
index a5fa44699..ce6110c0b 100644
--- a/pfsimulator/parflow_lib/problem_geometries.c
+++ b/pfsimulator/parflow_lib/problem_geometries.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *****************************************************************************/
@@ -360,7 +360,7 @@ PFModule   *GeometriesNewPublicXtra()
             NA_IndexToName(geom_input_na, i));
     intype_name = GetString(key);
     intype = NA_NameToIndexExitOnError(switch_na, intype_name, key);
-    
+
     num_new_solids = 0;
 
     switch (intype)
diff --git a/pfsimulator/parflow_lib/problem_heat_capacity.c b/pfsimulator/parflow_lib/problem_heat_capacity.c
index d9a8a1c1a..48ded7e39 100644
--- a/pfsimulator/parflow_lib/problem_heat_capacity.c
+++ b/pfsimulator/parflow_lib/problem_heat_capacity.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -50,7 +50,7 @@ typedef struct {
 /*--------------------------------------------------------------------------
  * HeatCapacity
  *--------------------------------------------------------------------------*/
-void         HeatCapacity(problem_data, heat_capacity)
+void HeatCapacity(problem_data, heat_capacity)
 Vector * heat_capacity;
 ProblemData *problem_data;
 {
diff --git a/pfsimulator/parflow_lib/problem_ic_phase_concen.c b/pfsimulator/parflow_lib/problem_ic_phase_concen.c
index d0609aa49..5e653e28b 100644
--- a/pfsimulator/parflow_lib/problem_ic_phase_concen.c
+++ b/pfsimulator/parflow_lib/problem_ic_phase_concen.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
diff --git a/pfsimulator/parflow_lib/problem_ic_phase_pressure.c b/pfsimulator/parflow_lib/problem_ic_phase_pressure.c
index c2ac3b96a..abbdf7d27 100644
--- a/pfsimulator/parflow_lib/problem_ic_phase_pressure.c
+++ b/pfsimulator/parflow_lib/problem_ic_phase_pressure.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 #include "parflow_netcdf.h"
@@ -91,7 +91,8 @@ typedef struct {
 } Type3;                      /* Spatially varying field over entire domain
                                * read from a file */
 typedef struct {
-  char    *ncFileName;
+  char    *filename;
+  int timestep;
   Vector  *ic_values;
 } Type4;                      /* Spatially varying field over entire domain
                                * read from a file */
@@ -120,12 +121,6 @@ void         ICPhasePressure(
 
   GrGeomSolid   *gr_solid, *gr_domain;
 
-  Type0         *dummy0;
-  Type1         *dummy1;
-  Type2         *dummy2;
-  Type3         *dummy3;
-  Type4         *dummy4;
-
   SubgridArray  *subgrids = GridSubgrids(grid);
 
   Subgrid       *subgrid;
@@ -192,7 +187,7 @@ void         ICPhasePressure(
       double  *values;
       int ir;
 
-      dummy0 = (Type0*)(public_xtra->data);
+      Type0* dummy0 = (Type0*)(public_xtra->data);
 
       num_regions = (dummy0->num_regions);
       region_indices = (dummy0->region_indices);
@@ -252,7 +247,7 @@ void         ICPhasePressure(
       double nonlin_resid;
       int ir;
 
-      dummy1 = (Type1*)(public_xtra->data);
+      Type1* dummy1 = (Type1*)(public_xtra->data);
 
       num_regions = (dummy1->num_regions);
       region_indices = (dummy1->region_indices);
@@ -463,7 +458,7 @@ void         ICPhasePressure(
       double  ***elevations;
       int ir;
 
-      dummy2 = (Type2*)(public_xtra->data);
+      Type2* dummy2 = (Type2*)(public_xtra->data);
 
       num_regions = (dummy2->num_regions);
       region_indices = (dummy2->region_indices);
@@ -678,7 +673,7 @@ void         ICPhasePressure(
 
     case 3: /* ParFlow binary file with spatially varying pressure values */
     {
-      dummy3 = (Type3*)(public_xtra->data);
+      Type3* dummy3 = (Type3*)(public_xtra->data);
 
       Vector *ic_values = dummy3->ic_values;
 
@@ -724,7 +719,7 @@ void         ICPhasePressure(
 
     case 4: /* ParFlow NetCDF file with spatially varying pressure values */
     {
-      dummy4 = (Type4*)(public_xtra->data);
+      Type4         *dummy4 = (Type4*)(public_xtra->data);
 
       Vector *ic_values = dummy4->ic_values;
 
@@ -792,8 +787,6 @@ PFModule  *ICPhasePressureInitInstanceXtra(
   PublicXtra    *public_xtra = (PublicXtra*)PFModulePublicXtra(this_module);
   InstanceXtra  *instance_xtra;
 
-  Type3         *dummy3;
-  Type4         *dummy4;
 
   if (PFModuleInstanceXtra(this_module) == NULL)
     instance_xtra = ctalloc(InstanceXtra, 1);
@@ -814,7 +807,7 @@ PFModule  *ICPhasePressureInitInstanceXtra(
     /* Uses a spatially varying field */
     if (public_xtra->type == 3)
     {
-      dummy3 = (Type3*)(public_xtra->data);
+      Type3* dummy3 = (Type3*)(public_xtra->data);
 
       /* Allocate temp vector */
       dummy3->ic_values = NewVectorType(grid, 1, 1, vector_cell_centered);
@@ -822,11 +815,11 @@ PFModule  *ICPhasePressureInitInstanceXtra(
     }
     else if (public_xtra->type == 4)
     {
-      dummy4 = (Type4*)(public_xtra->data);
+      Type4* dummy4 = (Type4*)(public_xtra->data);
 
       /* Allocate temp vector */
       dummy4->ic_values = NewVectorType(grid, 1, 1, vector_cell_centered);
-      ReadPFNC(dummy4->ncFileName, dummy4->ic_values, "pressure", 0, 3);
+      ReadPFNC(dummy4->filename, dummy4->ic_values, "pressure", dummy4->timestep, 3);
     }
   }
 
@@ -862,19 +855,16 @@ void  ICPhasePressureFreeInstanceXtra()
 
   if (instance_xtra)
   {
-    Type3         *dummy3;
-    Type4         *dummy4;
-
     /* Uses a spatially varying field */
     if (public_xtra->type == 3)
     {
-      dummy3 = (Type3*)(public_xtra->data);
+      Type3* dummy3 = (Type3*)(public_xtra->data);
 
       FreeVector(dummy3->ic_values);
     }
     else if (public_xtra->type == 4)
     {
-      dummy4 = (Type4*)(public_xtra->data);
+      Type4* dummy4 = (Type4*)(public_xtra->data);
       FreeVector(dummy4->ic_values);
     }
 
@@ -896,12 +886,6 @@ PFModule   *ICPhasePressureNewPublicXtra()
   int num_regions;
   int ir;
 
-  Type0         *dummy0;
-  Type1         *dummy1;
-  Type2         *dummy2;
-  Type3         *dummy3;
-  Type4         *dummy4;
-
   char *switch_name;
   char *region;
 
@@ -928,7 +912,7 @@ PFModule   *ICPhasePressureNewPublicXtra()
   {
     case 0:
     {
-      dummy0 = ctalloc(Type0, 1);
+      Type0* dummy0 = ctalloc(Type0, 1);
 
       dummy0->num_regions = num_regions;
 
@@ -952,7 +936,7 @@ PFModule   *ICPhasePressureNewPublicXtra()
 
     case 1:
     {
-      dummy1 = ctalloc(Type1, 1);
+      Type1* dummy1 = ctalloc(Type1, 1);
 
       dummy1->num_regions = num_regions;
 
@@ -981,7 +965,7 @@ PFModule   *ICPhasePressureNewPublicXtra()
 
     case 2:
     {
-      dummy2 = ctalloc(Type2, 1);
+      Type2* dummy2 = ctalloc(Type2, 1);
 
       dummy2->num_regions = num_regions;
 
@@ -1002,15 +986,14 @@ PFModule   *ICPhasePressureNewPublicXtra()
         sprintf(key, "Geom.%s.ICPressure.RefGeom", region);
         switch_name = GetString(key);
         dummy2->geom_indices[ir] = NA_NameToIndexExitOnError(GlobalsGeomNames,
-							     switch_name, key);
+                                                             switch_name, key);
 
         sprintf(key, "Geom.%s.ICPressure.RefPatch", region);
         switch_name = GetString(key);
         dummy2->patch_indices[ir] =
           NA_NameToIndexExitOnError(GeomSolidPatches(
-                                          GlobalsGeometries[dummy2->geom_indices[ir]]),
-				    switch_name, key);
-
+                                                     GlobalsGeometries[dummy2->geom_indices[ir]]),
+                                    switch_name, key);
       }
 
       (public_xtra->data) = (void*)dummy2;
@@ -1020,7 +1003,7 @@ PFModule   *ICPhasePressureNewPublicXtra()
 
     case 3:
     {
-      dummy3 = ctalloc(Type3, 1);
+      Type3* dummy3 = ctalloc(Type3, 1);
 
       sprintf(key, "Geom.%s.ICPressure.FileName", "domain");
       dummy3->filename = GetString(key);
@@ -1031,10 +1014,13 @@ PFModule   *ICPhasePressureNewPublicXtra()
 
     case 4:
     {
-      dummy4 = ctalloc(Type4, 1);
+      Type4* dummy4 = ctalloc(Type4, 1);
 
       sprintf(ncKey, "Geom.%s.ICPressure.FileName", "domain");
-      dummy4->ncFileName = GetString(ncKey);
+      dummy4->filename = GetString(ncKey);
+
+      sprintf(ncKey, "Geom.%s.ICPressure.TimeStep", "domain");
+      dummy4->timestep = GetIntDefault(ncKey, 0);
 
       public_xtra->data = (void*)dummy4;
 
@@ -1063,12 +1049,6 @@ void  ICPhasePressureFreePublicXtra()
   PublicXtra  *public_xtra = (PublicXtra*)PFModulePublicXtra(this_module);
 
 
-  Type0       *dummy0;
-  Type1       *dummy1;
-  Type2       *dummy2;
-  Type3       *dummy3;
-  Type4       *dummy4;
-
   if (public_xtra)
   {
     NA_FreeNameArray(public_xtra->regions);
@@ -1077,7 +1057,7 @@ void  ICPhasePressureFreePublicXtra()
     {
       case 0:
       {
-        dummy0 = (Type0*)(public_xtra->data);
+        Type0* dummy0 = (Type0*)(public_xtra->data);
 
         tfree(dummy0->region_indices);
         tfree(dummy0->values);
@@ -1087,7 +1067,7 @@ void  ICPhasePressureFreePublicXtra()
 
       case 1:
       {
-        dummy1 = (Type1*)(public_xtra->data);
+        Type1* dummy1 = (Type1*)(public_xtra->data);
 
         tfree(dummy1->region_indices);
         tfree(dummy1->reference_elevations);
@@ -1098,7 +1078,7 @@ void  ICPhasePressureFreePublicXtra()
 
       case 2:
       {
-        dummy2 = (Type2*)(public_xtra->data);
+        Type2* dummy2 = (Type2*)(public_xtra->data);
 
         tfree(dummy2->region_indices);
         tfree(dummy2->patch_indices);
@@ -1110,14 +1090,14 @@ void  ICPhasePressureFreePublicXtra()
 
       case 3:
       {
-        dummy3 = (Type3*)(public_xtra->data);
+        Type3* dummy3 = (Type3*)(public_xtra->data);
         tfree(dummy3);
         break;
       }
 
       case 4:
       {
-        dummy4 = (Type4*)(public_xtra->data);
+        Type4* dummy4 = (Type4*)(public_xtra->data);
         tfree(dummy4);
         break;
       }
diff --git a/pfsimulator/parflow_lib/problem_ic_phase_satur.c b/pfsimulator/parflow_lib/problem_ic_phase_satur.c
index 6462ad950..c45835019 100644
--- a/pfsimulator/parflow_lib/problem_ic_phase_satur.c
+++ b/pfsimulator/parflow_lib/problem_ic_phase_satur.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -272,7 +272,7 @@ PFModule   *ICPhaseSaturNewPublicXtra(
 
       default:
       {
-            InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+        InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
       }
     }
   }
diff --git a/pfsimulator/parflow_lib/problem_mannings.c b/pfsimulator/parflow_lib/problem_mannings.c
index 008980906..7b14301a3 100644
--- a/pfsimulator/parflow_lib/problem_mannings.c
+++ b/pfsimulator/parflow_lib/problem_mannings.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -478,7 +478,8 @@ PFModule  *ManningsNewPublicXtra()
 
     default:
     {
-      InputError("Invalid switch value <%s> for key <%s>", switch_name, key);    }
+      InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+    }
   }       /* End case statement */
 
 
diff --git a/pfsimulator/parflow_lib/problem_phase_density.c b/pfsimulator/parflow_lib/problem_phase_density.c
index 7a3719313..16c86b2da 100644
--- a/pfsimulator/parflow_lib/problem_phase_density.c
+++ b/pfsimulator/parflow_lib/problem_phase_density.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *  This module computes phase densities.  Currently, two types of densities
@@ -77,9 +77,9 @@ typedef struct {
  * Used to get the constant values for when density vector is always NULL
  *-------------------------------------------------------------------------*/
 
-void    PhaseDensityConstants(int phase,
-                              int fcn,
-                              int *phase_type,
+void    PhaseDensityConstants(int     phase,
+                              int     fcn,
+                              int *   phase_type,
                               double *constant,
                               double *ref_den,
                               double *comp_const)
@@ -90,13 +90,16 @@ void    PhaseDensityConstants(int phase,
   Type1 *dummy1;
 
   (*phase_type) = public_xtra->type[phase];
-  switch(*phase_type)
+  switch (*phase_type)
   {
     case 0:
-      if (fcn == CALCFCN) {
+      if (fcn == CALCFCN)
+      {
         dummy0 = (Type0*)(public_xtra->data[phase]);
         (*constant) = dummy0->constant;
-      } else {
+      }
+      else
+      {
         (*constant) = 0.0;
       }
       break;
@@ -409,7 +412,7 @@ PFModule  *PhaseDensityNewPublicXtra(
 
       default:
       {
-	InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+        InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
       }
     }
   }
diff --git a/pfsimulator/parflow_lib/problem_phase_heat_capacity.c b/pfsimulator/parflow_lib/problem_phase_heat_capacity.c
index 5a02b8848..07e9e637f 100644
--- a/pfsimulator/parflow_lib/problem_phase_heat_capacity.c
+++ b/pfsimulator/parflow_lib/problem_phase_heat_capacity.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -254,7 +254,7 @@ PFModule   *PhaseHeatCapacityNewPublicXtra(
 
       default:
       {
-            InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+        InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
       }
     }
   }  /*End over phases */
diff --git a/pfsimulator/parflow_lib/problem_phase_internal_energy.c b/pfsimulator/parflow_lib/problem_phase_internal_energy.c
index 8b91ece66..ee4f01a39 100644
--- a/pfsimulator/parflow_lib/problem_phase_internal_energy.c
+++ b/pfsimulator/parflow_lib/problem_phase_internal_energy.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *  This module computes phase densities.  Currently, two types of densities
@@ -439,7 +439,7 @@ PFModule  *InternalEnergyDensityNewPublicXtra(
 
       default:
       {
-	InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+        InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
       }
     }
   }
@@ -476,7 +476,8 @@ PFModule  *InternalEnergyDensityNewPublicXtra(
 
       default:
       {
-	InputError("Invalid switch value <%s> for key <%s>", switch_name, key);      }
+        InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+      }
     }
   }
 
diff --git a/pfsimulator/parflow_lib/problem_phase_mobility.c b/pfsimulator/parflow_lib/problem_phase_mobility.c
index 9089e013b..091c0e122 100644
--- a/pfsimulator/parflow_lib/problem_phase_mobility.c
+++ b/pfsimulator/parflow_lib/problem_phase_mobility.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *****************************************************************************/
@@ -344,7 +344,7 @@ PFModule  *PhaseMobilityNewPublicXtra(
 
       default:
       {
-	InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+        InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
       }
     }
   }
diff --git a/pfsimulator/parflow_lib/problem_phase_rel_perm.c b/pfsimulator/parflow_lib/problem_phase_rel_perm.c
index 00fc35122..6894bf461 100644
--- a/pfsimulator/parflow_lib/problem_phase_rel_perm.c
+++ b/pfsimulator/parflow_lib/problem_phase_rel_perm.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -198,7 +198,7 @@ VanGTable *VanGComputeTable(
 
   // GCC 11.3.0 WITH optimization warns that del is possibly used without initialization.
   // This silences the warning.   Looks like a false warning, what is the opt doing?
-  memset(del, num_sample_points+1, sizeof(double));
+  memset(del, num_sample_points + 1, sizeof(double));
   // begin monotonic spline (see Fritsch and Carlson, SIAM J. Num. Anal., 17 (2), 1980)
   for (index = 0; index < num_sample_points; index++)
   {
@@ -363,6 +363,7 @@ static inline double VanGLookupLinear(
   int num_sample_points = lookup_table->num_sample_points;
   double min_pressure_head = lookup_table->min_pressure_head;
   int max = num_sample_points + 1;
+
   PF_UNUSED(max);
 
   // This table goes from 0 to fabs(min_pressure_head)
@@ -503,7 +504,7 @@ void         PhaseRelPerm(
                            nx, ny, nz,
             {
               int ipr = SubvectorEltIndex(pr_sub,
-                                      i + fdir[0], j + fdir[1], k + fdir[2]);
+                                          i + fdir[0], j + fdir[1], k + fdir[2]);
               prdat[ipr] = values[ir];
             });
           }
@@ -513,7 +514,7 @@ void         PhaseRelPerm(
                            nx, ny, nz,
             {
               int ipr = SubvectorEltIndex(pr_sub,
-                                      i + fdir[0], j + fdir[1], k + fdir[2]);
+                                          i + fdir[0], j + fdir[1], k + fdir[2]);
               prdat[ipr] = 0.0;
             });
           }     /* End else clause */
@@ -692,11 +693,11 @@ void         PhaseRelPerm(
                       /* Table Lookup */
 
                       int ipr = SubvectorEltIndex(pr_sub,
-                                              i + fdir[0], j + fdir[1], k + fdir[2]);
+                                                  i + fdir[0], j + fdir[1], k + fdir[2]);
                       int ipp = SubvectorEltIndex(pp_sub,
-                                              i + fdir[0], j + fdir[1], k + fdir[2]);
+                                                  i + fdir[0], j + fdir[1], k + fdir[2]);
                       int ipd = SubvectorEltIndex(pd_sub,
-                                              i + fdir[0], j + fdir[1], k + fdir[2]);
+                                                  i + fdir[0], j + fdir[1], k + fdir[2]);
 
                       if (ppdat[ipp] >= 0.0)
                         prdat[ipr] = 1.0;
@@ -720,7 +721,7 @@ void         PhaseRelPerm(
                     double min_pressure_head = lookup_table->min_pressure_head;
                     int num_sample_points = lookup_table->num_sample_points;
                     int max = num_sample_points + 1;
-		    PF_UNUSED(max);
+                    PF_UNUSED(max);
 
                     GrGeomSurfLoop(i, j, k, fdir, gr_solid, r, ix, iy, iz,
                                    nx, ny, nz,
@@ -728,11 +729,11 @@ void         PhaseRelPerm(
                       /* Table Lookup */
 
                       int ipr = SubvectorEltIndex(pr_sub,
-                                              i + fdir[0], j + fdir[1], k + fdir[2]);
+                                                  i + fdir[0], j + fdir[1], k + fdir[2]);
                       int ipp = SubvectorEltIndex(pp_sub,
-                                              i + fdir[0], j + fdir[1], k + fdir[2]);
+                                                  i + fdir[0], j + fdir[1], k + fdir[2]);
                       int ipd = SubvectorEltIndex(pd_sub,
-                                              i + fdir[0], j + fdir[1], k + fdir[2]);
+                                                  i + fdir[0], j + fdir[1], k + fdir[2]);
 
                       if (ppdat[ipp] >= 0.0)
                         prdat[ipr] = 1.0;
@@ -766,11 +767,11 @@ void         PhaseRelPerm(
                                nx, ny, nz,
                 {
                   int ipr = SubvectorEltIndex(pr_sub,
-                                          i + fdir[0], j + fdir[1], k + fdir[2]);
+                                              i + fdir[0], j + fdir[1], k + fdir[2]);
                   int ipp = SubvectorEltIndex(pp_sub,
-                                          i + fdir[0], j + fdir[1], k + fdir[2]);
+                                              i + fdir[0], j + fdir[1], k + fdir[2]);
                   int ipd = SubvectorEltIndex(pd_sub,
-                                          i + fdir[0], j + fdir[1], k + fdir[2]);
+                                              i + fdir[0], j + fdir[1], k + fdir[2]);
 
                   if (ppdat[ipp] >= 0.0)
                     prdat[ipr] = 1.0;
@@ -804,11 +805,11 @@ void         PhaseRelPerm(
                       /* Table Lookup */
 
                       int ipr = SubvectorEltIndex(pr_sub,
-                                              i + fdir[0], j + fdir[1], k + fdir[2]);
+                                                  i + fdir[0], j + fdir[1], k + fdir[2]);
                       int ipp = SubvectorEltIndex(pp_sub,
-                                              i + fdir[0], j + fdir[1], k + fdir[2]);
+                                                  i + fdir[0], j + fdir[1], k + fdir[2]);
                       int ipd = SubvectorEltIndex(pd_sub,
-                                              i + fdir[0], j + fdir[1], k + fdir[2]);
+                                                  i + fdir[0], j + fdir[1], k + fdir[2]);
 
                       if (ppdat[ipp] >= 0.0)
                         prdat[ipr] = 0.0;
@@ -831,7 +832,7 @@ void         PhaseRelPerm(
                     double min_pressure_head = lookup_table->min_pressure_head;
                     int num_sample_points = lookup_table->num_sample_points;
                     int max = num_sample_points + 1;
-		    PF_UNUSED(max);
+                    PF_UNUSED(max);
 
                     GrGeomSurfLoop(i, j, k, fdir, gr_solid, r, ix, iy, iz,
                                    nx, ny, nz,
@@ -839,11 +840,11 @@ void         PhaseRelPerm(
                       /* Table Lookup */
 
                       int ipr = SubvectorEltIndex(pr_sub,
-                                              i + fdir[0], j + fdir[1], k + fdir[2]);
+                                                  i + fdir[0], j + fdir[1], k + fdir[2]);
                       int ipp = SubvectorEltIndex(pp_sub,
-                                              i + fdir[0], j + fdir[1], k + fdir[2]);
+                                                  i + fdir[0], j + fdir[1], k + fdir[2]);
                       int ipd = SubvectorEltIndex(pd_sub,
-                                              i + fdir[0], j + fdir[1], k + fdir[2]);
+                                                  i + fdir[0], j + fdir[1], k + fdir[2]);
 
                       if (ppdat[ipp] >= 0.0)
                         prdat[ipr] = 0.0;
@@ -883,11 +884,11 @@ void         PhaseRelPerm(
                                nx, ny, nz,
                 {
                   int ipr = SubvectorEltIndex(pr_sub,
-                                          i + fdir[0], j + fdir[1], k + fdir[2]);
+                                              i + fdir[0], j + fdir[1], k + fdir[2]);
                   int ipp = SubvectorEltIndex(pp_sub,
-                                          i + fdir[0], j + fdir[1], k + fdir[2]);
+                                              i + fdir[0], j + fdir[1], k + fdir[2]);
                   int ipd = SubvectorEltIndex(pd_sub,
-                                          i + fdir[0], j + fdir[1], k + fdir[2]);
+                                              i + fdir[0], j + fdir[1], k + fdir[2]);
 
                   if (ppdat[ipp] >= 0.0)
                     prdat[ipr] = 0.0;
@@ -956,11 +957,11 @@ void         PhaseRelPerm(
                            nx, ny, nz,
             {
               int ipr = SubvectorEltIndex(pr_sub,
-                                      i + fdir[0], j + fdir[1], k + fdir[2]);
+                                          i + fdir[0], j + fdir[1], k + fdir[2]);
               int ipp = SubvectorEltIndex(pp_sub,
-                                      i + fdir[0], j + fdir[1], k + fdir[2]);
+                                          i + fdir[0], j + fdir[1], k + fdir[2]);
               int ipd = SubvectorEltIndex(pd_sub,
-                                      i + fdir[0], j + fdir[1], k + fdir[2]);
+                                          i + fdir[0], j + fdir[1], k + fdir[2]);
 
               int n_index = SubvectorEltIndex(n_values_sub, i, j, k);
               int alpha_index = SubvectorEltIndex(alpha_values_sub, i, j, k);
@@ -987,11 +988,11 @@ void         PhaseRelPerm(
                            nx, ny, nz,
             {
               int ipr = SubvectorEltIndex(pr_sub,
-                                      i + fdir[0], j + fdir[1], k + fdir[2]);
+                                          i + fdir[0], j + fdir[1], k + fdir[2]);
               int ipp = SubvectorEltIndex(pp_sub,
-                                      i + fdir[0], j + fdir[1], k + fdir[2]);
+                                          i + fdir[0], j + fdir[1], k + fdir[2]);
               int ipd = SubvectorEltIndex(pd_sub,
-                                      i + fdir[0], j + fdir[1], k + fdir[2]);
+                                          i + fdir[0], j + fdir[1], k + fdir[2]);
 
               int n_index = SubvectorEltIndex(n_values_sub, i, j, k);
               int alpha_index = SubvectorEltIndex(alpha_values_sub, i, j, k);
@@ -1087,7 +1088,7 @@ void         PhaseRelPerm(
                     double min_pressure_head = lookup_table->min_pressure_head;
                     int num_sample_points = lookup_table->num_sample_points;
                     int max = num_sample_points + 1;
-		    PF_UNUSED(max);
+                    PF_UNUSED(max);
 
                     GrGeomInLoop(i, j, k, gr_solid, r, ix, iy, iz, nx, ny, nz,
                     {
@@ -1190,7 +1191,7 @@ void         PhaseRelPerm(
                     double min_pressure_head = lookup_table->min_pressure_head;
                     int num_sample_points = lookup_table->num_sample_points;
                     int max = num_sample_points + 1;
-		    PF_UNUSED(max);
+                    PF_UNUSED(max);
 
                     GrGeomInLoop(i, j, k, gr_solid, r, ix, iy, iz, nx, ny, nz,
                     {
@@ -1402,11 +1403,11 @@ void         PhaseRelPerm(
                            nx, ny, nz,
             {
               int ipr = SubvectorEltIndex(pr_sub,
-                                      i + fdir[0], j + fdir[1], k + fdir[2]);
+                                          i + fdir[0], j + fdir[1], k + fdir[2]);
               int ipp = SubvectorEltIndex(pp_sub,
-                                      i + fdir[0], j + fdir[1], k + fdir[2]);
+                                          i + fdir[0], j + fdir[1], k + fdir[2]);
               int ipd = SubvectorEltIndex(pd_sub,
-                                      i + fdir[0], j + fdir[1], k + fdir[2]);
+                                          i + fdir[0], j + fdir[1], k + fdir[2]);
 
               if (ppdat[ipp] >= 0.0)
                 prdat[ipr] = 1.0;
@@ -1424,11 +1425,11 @@ void         PhaseRelPerm(
                            nx, ny, nz,
             {
               int ipr = SubvectorEltIndex(pr_sub,
-                                      i + fdir[0], j + fdir[1], k + fdir[2]);
+                                          i + fdir[0], j + fdir[1], k + fdir[2]);
               int ipp = SubvectorEltIndex(pp_sub,
-                                      i + fdir[0], j + fdir[1], k + fdir[2]);
+                                          i + fdir[0], j + fdir[1], k + fdir[2]);
               int ipd = SubvectorEltIndex(pd_sub,
-                                      i + fdir[0], j + fdir[1], k + fdir[2]);
+                                          i + fdir[0], j + fdir[1], k + fdir[2]);
 
               if (ppdat[ipp] >= 0.0)
                 prdat[ipr] = 0.0;
@@ -1567,9 +1568,9 @@ void         PhaseRelPerm(
                            nx, ny, nz,
             {
               int ipr = SubvectorEltIndex(pr_sub,
-                                      i + fdir[0], j + fdir[1], k + fdir[2]);
+                                          i + fdir[0], j + fdir[1], k + fdir[2]);
               int ipp = SubvectorEltIndex(pp_sub,
-                                      i + fdir[0], j + fdir[1], k + fdir[2]);
+                                          i + fdir[0], j + fdir[1], k + fdir[2]);
               if (ppdat[ipp] == 0.0)
                 prdat[ipr] = region_coeffs[0];
               else
@@ -1588,9 +1589,9 @@ void         PhaseRelPerm(
                            nx, ny, nz,
             {
               int ipr = SubvectorEltIndex(pr_sub,
-                                      i + fdir[0], j + fdir[1], k + fdir[2]);
+                                          i + fdir[0], j + fdir[1], k + fdir[2]);
               int ipp = SubvectorEltIndex(pp_sub,
-                                      i + fdir[0], j + fdir[1], k + fdir[2]);
+                                          i + fdir[0], j + fdir[1], k + fdir[2]);
               if (ppdat[ipp] == 0.0)
                 prdat[ipr] = 0.0;
               else
diff --git a/pfsimulator/parflow_lib/problem_phase_source.c b/pfsimulator/parflow_lib/problem_phase_source.c
index b26edefa7..55cbbbd6b 100644
--- a/pfsimulator/parflow_lib/problem_phase_source.c
+++ b/pfsimulator/parflow_lib/problem_phase_source.c
@@ -1,14 +1,32 @@
-/*BHEADER*********************************************************************
-* (c) 1995   The Regents of the University of California
+/*BHEADER**********************************************************************
 *
-* See the file COPYRIGHT_and_DISCLAIMER for a complete copyright
-* notice, contact person, and disclaimer.
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
 *
-* $Revision: 1.23 $
-*********************************************************************EHEADER*/
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
-
 #include <float.h>
 
 /*--------------------------------------------------------------------------
@@ -56,6 +74,9 @@ void         PhaseSource(
   WellDataPhysical *well_data_physical;
   WellDataValue    *well_data_value;
 
+  ReservoirData         *reservoir_data = ProblemDataReservoirData(problem_data);
+  ReservoirDataPhysical *reservoir_data_physical;
+
   TimeCycleData    *time_cycle_data;
 
   Vector           *perm_x = ProblemDataPermeabilityX(problem_data);
@@ -69,7 +90,7 @@ void         PhaseSource(
 
   SubgridArray     *subgrids = GridSubgrids(grid);
 
-  Subgrid          *subgrid, *well_subgrid, *tmp_subgrid;
+  Subgrid          *subgrid, *well_subgrid, *tmp_subgrid, *reservoir_release_subgrid;
   Subvector        *px_sub, *py_sub, *pz_sub, *ps_sub;
 
   double           *data, *px, *py, *pz;
@@ -83,10 +104,10 @@ void         PhaseSource(
 
   int is, i, j, k;
 
-  /* Locals associated with wells */
+  /* Locals associated with wells and reservoirs*/
   int well;
   int cycle_number, interval_number;
-  double volume, flux, well_value;
+  double volume, flux, well_value, reservoir_value;
 
 // SGS FIXME why is this needed?
 #undef max
@@ -351,7 +372,7 @@ void         PhaseSource(
           data = SubvectorElt(ps_sub, ix, iy, iz);
 
           int ip = 0;
-          int ips = 0;          
+          int ips = 0;
 
           if (WellDataPhysicalMethod(well_data_physical)
               == FLUX_WEIGHTED)
@@ -361,16 +382,20 @@ void         PhaseSource(
                       ips, nx_ps, ny_ps, nz_ps, 1, 1, 1,
             {
               double weight = (px[ip] / avg_x) * (area_x / area_sum)
-                      + (py[ip] / avg_y) * (area_y / area_sum)
-                      + (pz[ip] / avg_z) * (area_z / area_sum);
+                              + (py[ip] / avg_y) * (area_y / area_sum)
+                              + (pz[ip] / avg_z) * (area_z / area_sum);
               data[ips] += weight * flux;
             });
-          }else{
-            double weight = -FLT_MAX;            
+          }
+          else
+          {
+            double weight = -FLT_MAX;
             if (WellDataPhysicalMethod(well_data_physical)
-                == FLUX_STANDARD)weight = 1.0;
+                == FLUX_STANDARD)
+              weight = 1.0;
             else if (WellDataPhysicalMethod(well_data_physical)
-                     == FLUX_PATTERNED)weight = 0.0;
+                     == FLUX_PATTERNED)
+              weight = 0.0;
             BoxLoopI2(i, j, k, ix, iy, iz, nx, ny, nz,
                       ip, nx_p, ny_p, nz_p, 1, 1, 1,
                       ips, nx_ps, ny_ps, nz_ps, 1, 1, 1,
@@ -378,15 +403,89 @@ void         PhaseSource(
               data[ips] += weight * flux;
             });
           }
-          /* done with this temporay subgrid */
+          /* done with this temporary subgrid */
           FreeSubgrid(tmp_subgrid);
         }
       }
     }
   }  /* End well data */
+
+  if (ReservoirDataNumReservoirs(reservoir_data) > 0)
+  {
+    for (int reservoir = 0; reservoir < ReservoirDataNumReservoirs(reservoir_data); reservoir++)
+    {
+      reservoir_data_physical = ReservoirDataReservoirPhysical(reservoir_data, reservoir);
+      reservoir_release_subgrid = ReservoirDataPhysicalReleaseSubgrid(reservoir_data_physical);
+      reservoir_value = ReservoirDataPhysicalReleaseRate(reservoir_data_physical);
+      volume = ReservoirDataPhysicalSize(reservoir_data_physical);
+      flux = reservoir_value / (volume);
+      //If we are overfull need to release the rest of the flux
+      if (reservoir_data_physical->storage > reservoir_data_physical->max_storage)
+      {
+        flux = (reservoir_data_physical->storage - reservoir_data_physical->max_storage) / (volume);
+      }
+      ForSubgridI(is, subgrids)
+      {
+        subgrid = SubgridArraySubgrid(subgrids, is);
+
+        px_sub = VectorSubvector(perm_x, is);
+        py_sub = VectorSubvector(perm_y, is);
+        pz_sub = VectorSubvector(perm_z, is);
+
+        ps_sub = VectorSubvector(phase_source, is);
+
+        nx_p = SubvectorNX(ps_sub);
+        ny_p = SubvectorNY(ps_sub);
+        nz_p = SubvectorNZ(ps_sub);
+
+        nx_ps = SubvectorNX(ps_sub);
+        ny_ps = SubvectorNY(ps_sub);
+        nz_ps = SubvectorNZ(ps_sub);
+
+        if (reservoir_data_physical->storage > reservoir_data_physical->min_release_storage)
+        {
+          reservoir_data_physical = ReservoirDataReservoirPhysical(reservoir_data, reservoir);
+          /*  Get the intersection of the reservoir with the subgrid  */
+          if ((tmp_subgrid = IntersectSubgrids(subgrid, reservoir_release_subgrid)))
+          {
+            /*  If an intersection;  loop over it, and insert value  */
+            ix = SubgridIX(tmp_subgrid);
+            iy = SubgridIY(tmp_subgrid);
+            iz = SubgridIZ(tmp_subgrid);
+
+            nx = SubgridNX(tmp_subgrid);
+            ny = SubgridNY(tmp_subgrid);
+            nz = SubgridNZ(tmp_subgrid);
+
+            dx = SubgridDX(tmp_subgrid);
+            dy = SubgridDY(tmp_subgrid);
+            dz = SubgridDZ(tmp_subgrid);
+
+            px = SubvectorElt(px_sub, ix, iy, iz);
+            py = SubvectorElt(py_sub, ix, iy, iz);
+            pz = SubvectorElt(pz_sub, ix, iy, iz);
+
+            data = SubvectorElt(ps_sub, ix, iy, iz);
+
+            int ips = 0;
+            double weight = 1.0;
+
+            BoxLoopI1(i, j, k, ix, iy, iz, nx, ny, nz,
+                      ips, nx_ps, ny_ps, nz_ps, 1, 1, 1,
+            {
+              data[ips] += weight * flux;
+              ReservoirDataPhysicalReleaseAmountInSolver(reservoir_data_physical) = flux * volume;
+            });
+          }
+        }
+      }
+    }
+  }
 }
 
 
+
+
 /*--------------------------------------------------------------------------
  * PhaseSourceInitInstanceXtra
  *--------------------------------------------------------------------------*/
@@ -523,7 +622,7 @@ PFModule  *PhaseSourceNewPublicXtra(
 
       default:
       {
-	InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+        InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
       }
     }     /* End case statement */
   }
diff --git a/pfsimulator/parflow_lib/problem_phase_source_with_initxtra.c b/pfsimulator/parflow_lib/problem_phase_source_with_initxtra.c
index 89fb7b47c..bc6922041 100644
--- a/pfsimulator/parflow_lib/problem_phase_source_with_initxtra.c
+++ b/pfsimulator/parflow_lib/problem_phase_source_with_initxtra.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -65,7 +65,7 @@ typedef struct {
  * PhaseSource
  *--------------------------------------------------------------------------*/
 
-void         PhaseSource(phase_source, problem, problem_data, time)
+void PhaseSource(phase_source, problem, problem_data, time)
 Vector * phase_source;
 Problem     *problem;
 ProblemData *problem_data;
diff --git a/pfsimulator/parflow_lib/problem_phase_viscosity.c b/pfsimulator/parflow_lib/problem_phase_viscosity.c
index 6168e0aec..40898ef86 100644
--- a/pfsimulator/parflow_lib/problem_phase_viscosity.c
+++ b/pfsimulator/parflow_lib/problem_phase_viscosity.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *  This module computes phase densities.  Currently, two types of densities
@@ -346,7 +346,7 @@ PFModule  *PhaseViscosityNewPublicXtra(
 
       default:
       {
-	InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+        InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
       }
     }
   }
diff --git a/pfsimulator/parflow_lib/problem_porosity.c b/pfsimulator/parflow_lib/problem_porosity.c
index 7c6799214..4ff625c4a 100644
--- a/pfsimulator/parflow_lib/problem_porosity.c
+++ b/pfsimulator/parflow_lib/problem_porosity.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *****************************************************************************/
@@ -82,8 +82,8 @@ void Porosity(
   SubgridArray     *subgrids;
 
   Subgrid          *subgrid,
-    *well_subgrid,
-    *tmp_subgrid;
+                   *well_subgrid,
+                   *tmp_subgrid;
 
   Subvector        *subvector;
 
@@ -369,7 +369,7 @@ PFModule   *PorosityNewPublicXtra()
 
       default:
       {
-	InputError("Invalid switch value <%s> for key <%s>", sim_type_name, key);
+        InputError("Invalid switch value <%s> for key <%s>", sim_type_name, key);
       }
     }
   }
diff --git a/pfsimulator/parflow_lib/problem_real_space_z.c b/pfsimulator/parflow_lib/problem_real_space_z.c
index 8ec2106e4..c051b9bac 100644
--- a/pfsimulator/parflow_lib/problem_real_space_z.c
+++ b/pfsimulator/parflow_lib/problem_real_space_z.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 #include "globals.h"
@@ -132,15 +132,15 @@ void realSpaceZ(ProblemData *problem_data, Vector *rsz)
         ips = SubvectorEltIndex(rsz_sub, i, j, k);
 
         breaking_out_PV_visiting = PV_visiting;
-	// Can't just do a break since GrGeom loops are multiple levels deep.
+        // Can't just do a break since GrGeom loops are multiple levels deep.
         goto  breakout;
       });
 
 breakout:;
 
-      /* 
-       * If we broke out of GrGeomInLoop we found point in 
-       * domain on this rank.   If not then nothing 
+      /*
+       * If we broke out of GrGeomInLoop we found point in
+       * domain on this rank.   If not then nothing
        * was in domain.
        */
 
diff --git a/pfsimulator/parflow_lib/problem_retardation.c b/pfsimulator/parflow_lib/problem_retardation.c
index 2cac79b0a..fb569d514 100644
--- a/pfsimulator/parflow_lib/problem_retardation.c
+++ b/pfsimulator/parflow_lib/problem_retardation.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -282,7 +282,7 @@ PFModule  *RetardationNewPublicXtra(
 
           default:
           {
-	    InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+            InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
           }
         }
       }
diff --git a/pfsimulator/parflow_lib/problem_richards_bc_internal.c b/pfsimulator/parflow_lib/problem_richards_bc_internal.c
index 8f36b0868..2d1379794 100644
--- a/pfsimulator/parflow_lib/problem_richards_bc_internal.c
+++ b/pfsimulator/parflow_lib/problem_richards_bc_internal.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
diff --git a/pfsimulator/parflow_lib/problem_saturation.c b/pfsimulator/parflow_lib/problem_saturation.c
index 3828da1d6..4139e617b 100644
--- a/pfsimulator/parflow_lib/problem_saturation.c
+++ b/pfsimulator/parflow_lib/problem_saturation.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -116,7 +116,7 @@ void     Saturation(
                                        *                  value
                                        * fcn = CALCDER => calculate the function
                                        *                  derivative */
-{ 
+{
   PFModule      *this_module = ThisPFModule;
   PublicXtra    *public_xtra = (PublicXtra*)PFModulePublicXtra(this_module);
 
@@ -693,10 +693,10 @@ PFModule  *SaturationInitInstanceXtra(
           FreeVector(dummy1->s_res_values);
           FreeVector(dummy1->s_sat_values);
 
-	  dummy1->n_values = NULL;
-	  dummy1->alpha_values = NULL;
-	  dummy1->s_res_values = NULL;
-	  dummy1->s_sat_values = NULL;
+          dummy1->n_values = NULL;
+          dummy1->alpha_values = NULL;
+          dummy1->s_res_values = NULL;
+          dummy1->s_sat_values = NULL;
         }
       }
       if (public_xtra->type == 5)
@@ -785,19 +785,19 @@ void  SaturationFreeInstanceXtra()
       Type1* dummy1 = (Type1*)(public_xtra->data);
       if ((dummy1->data_from_file) == 1)
       {
-	/* Data will be shared by all instances */
-	if (dummy1->n_values)
-	{
-	  FreeVector(dummy1->n_values);
-	  FreeVector(dummy1->alpha_values);
-	  FreeVector(dummy1->s_res_values);
-	  FreeVector(dummy1->s_sat_values);
-
-	  dummy1->n_values = NULL;
-	  dummy1->alpha_values = NULL;
-	  dummy1->s_res_values = NULL;
-	  dummy1->s_sat_values = NULL;
-	}
+        /* Data will be shared by all instances */
+        if (dummy1->n_values)
+        {
+          FreeVector(dummy1->n_values);
+          FreeVector(dummy1->alpha_values);
+          FreeVector(dummy1->s_res_values);
+          FreeVector(dummy1->s_sat_values);
+
+          dummy1->n_values = NULL;
+          dummy1->alpha_values = NULL;
+          dummy1->s_res_values = NULL;
+          dummy1->s_sat_values = NULL;
+        }
       }
     }
     if (public_xtra->type == 5)
@@ -1124,13 +1124,13 @@ void  SaturationFreePublicXtra()
         dummy1 = (Type1*)(public_xtra->data);
 
         if (dummy1->data_from_file == 0)
-	{
-	  tfree(dummy1->region_indices);
-	  tfree(dummy1->alphas);
-	  tfree(dummy1->ns);
-	  tfree(dummy1->s_ress);
-	  tfree(dummy1->s_difs);
-	}
+        {
+          tfree(dummy1->region_indices);
+          tfree(dummy1->alphas);
+          tfree(dummy1->ns);
+          tfree(dummy1->s_ress);
+          tfree(dummy1->s_difs);
+        }
 
         tfree(dummy1);
 
@@ -1232,9 +1232,9 @@ void  SaturationOutput()
 
 
 void  SaturationOutputStatic(
-			     char *file_prefix,
-			     ProblemData *problem_data /* Contains geometry info. for the problem */
-			     )
+                             char *       file_prefix,
+                             ProblemData *problem_data /* Contains geometry info. for the problem */
+                             )
 {
   PFModule      *this_module = ThisPFModule;
   PublicXtra    *public_xtra = (PublicXtra*)PFModulePublicXtra(this_module);
@@ -1278,7 +1278,7 @@ void  SaturationOutputStatic(
   Vector *pd_ssat = NewVectorType(grid, 1, 1, vector_cell_centered);
 
   /* Initialize saturations */
-  InitVector(pd_alpha, 0.0); 
+  InitVector(pd_alpha, 0.0);
   InitVector(pd_n, 0.0);
   InitVector(pd_sres, 0.0);
   InitVector(pd_ssat, 0.0);
@@ -1332,22 +1332,21 @@ void  SaturationOutputStatic(
             pd_sres_dat = SubvectorData(pd_sres_sub);     //BB
             pd_ssat_dat = SubvectorData(pd_ssat_sub);     //BB
 
-	    GrGeomInLoop(i, j, k, gr_solid, r, ix, iy, iz, nx, ny, nz,
-	    {
-	      int ips = SubvectorEltIndex(pd_alpha_sub, i, j, k);
-	      
-	      double alpha = alphas[ir];
-	      double n = ns[ir];
-	      double s_res = s_ress[ir];
-	      double s_dif = s_difs[ir];
-	      
-	      pd_alpha_dat[ips] = alpha;  //BB
-	      pd_n_dat[ips] = n;  //BB
-	      pd_sres_dat[ips] = s_res;  //BB  // no ssat???
-	      // Storing s_dif in the stucture, convert back to s_sat for output
-	      pd_ssat_dat[ips] = s_dif + s_res;
-	      
-	    });
+            GrGeomInLoop(i, j, k, gr_solid, r, ix, iy, iz, nx, ny, nz,
+            {
+              int ips = SubvectorEltIndex(pd_alpha_sub, i, j, k);
+
+              double alpha = alphas[ir];
+              double n = ns[ir];
+              double s_res = s_ress[ir];
+              double s_dif = s_difs[ir];
+
+              pd_alpha_dat[ips] = alpha;  //BB
+              pd_n_dat[ips] = n;  //BB
+              pd_sres_dat[ips] = s_res;  //BB  // no ssat???
+              // Storing s_dif in the stucture, convert back to s_sat for output
+              pd_ssat_dat[ips] = s_dif + s_res;
+            });
           }     /* End subgrid loop */
         }       /* End loop over regions */
       }         /* End if data not from file */
@@ -1393,25 +1392,25 @@ void  SaturationOutputStatic(
           pd_sres_dat = SubvectorData(pd_sres_sub);     //BB
           pd_ssat_dat = SubvectorData(pd_ssat_sub);     //BB
 
-	  GrGeomInLoop(i, j, k, gr_solid, r, ix, iy, iz, nx, ny, nz,
+          GrGeomInLoop(i, j, k, gr_solid, r, ix, iy, iz, nx, ny, nz,
           {
-	    int ips = SubvectorEltIndex(pd_alpha_sub, i, j, k);
-	    
-	    int n_index = SubvectorEltIndex(n_values_sub, i, j, k);
-	    int alpha_index = SubvectorEltIndex(alpha_values_sub, i, j, k);
-	    int s_res_index = SubvectorEltIndex(s_res_values_sub, i, j, k);
-	    int s_sat_index = SubvectorEltIndex(s_sat_values_sub, i, j, k);
-	    
-	    double alpha = alpha_values_dat[alpha_index];
-	    double n = n_values_dat[n_index];
-	    double s_res = s_res_values_dat[s_res_index];
-	    double s_sat = s_sat_values_dat[s_sat_index];
-	    
-	    pd_alpha_dat[ips] = alpha;  //BB
-	    pd_n_dat[ips] = n;  //BB
-	    pd_sres_dat[ips] = s_res;  //BB
-	    pd_ssat_dat[ips] = s_sat;  //BB
-	  });
+            int ips = SubvectorEltIndex(pd_alpha_sub, i, j, k);
+
+            int n_index = SubvectorEltIndex(n_values_sub, i, j, k);
+            int alpha_index = SubvectorEltIndex(alpha_values_sub, i, j, k);
+            int s_res_index = SubvectorEltIndex(s_res_values_sub, i, j, k);
+            int s_sat_index = SubvectorEltIndex(s_sat_values_sub, i, j, k);
+
+            double alpha = alpha_values_dat[alpha_index];
+            double n = n_values_dat[n_index];
+            double s_res = s_res_values_dat[s_res_index];
+            double s_sat = s_sat_values_dat[s_sat_index];
+
+            pd_alpha_dat[ips] = alpha;  //BB
+            pd_n_dat[ips] = n;  //BB
+            pd_sres_dat[ips] = s_res;  //BB
+            pd_ssat_dat[ips] = s_sat;  //BB
+          });
         }       /* End subgrid loop */
       }         /* End if data_from_file */
       break;
@@ -1419,22 +1418,22 @@ void  SaturationOutputStatic(
   }          /* End switch */
 
   char file_postfix[2048];
-  
+
   strcpy(file_postfix, "alpha");
   WritePFBinary(file_prefix, file_postfix,
-		pd_alpha);
-  
+                pd_alpha);
+
   strcpy(file_postfix, "n");
   WritePFBinary(file_prefix, file_postfix,
-		pd_n);
-  
+                pd_n);
+
   strcpy(file_postfix, "sres");
   WritePFBinary(file_prefix, file_postfix,
-		pd_sres);
-  
+                pd_sres);
+
   strcpy(file_postfix, "ssat");
   WritePFBinary(file_prefix, file_postfix,
-		pd_ssat);
+                pd_ssat);
 
   FreeVector(pd_alpha);
   FreeVector(pd_n);
diff --git a/pfsimulator/parflow_lib/problem_saturation_constitutive.c b/pfsimulator/parflow_lib/problem_saturation_constitutive.c
index 8c31bae00..53fd9b6b8 100644
--- a/pfsimulator/parflow_lib/problem_saturation_constitutive.c
+++ b/pfsimulator/parflow_lib/problem_saturation_constitutive.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *****************************************************************************/
diff --git a/pfsimulator/parflow_lib/problem_spec_storage.c b/pfsimulator/parflow_lib/problem_spec_storage.c
index 460d0f6b8..4bc1c761d 100644
--- a/pfsimulator/parflow_lib/problem_spec_storage.c
+++ b/pfsimulator/parflow_lib/problem_spec_storage.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
diff --git a/pfsimulator/parflow_lib/problem_thermal_conductivity.c b/pfsimulator/parflow_lib/problem_thermal_conductivity.c
index 36e32788c..b7765c4e0 100644
--- a/pfsimulator/parflow_lib/problem_thermal_conductivity.c
+++ b/pfsimulator/parflow_lib/problem_thermal_conductivity.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
diff --git a/pfsimulator/parflow_lib/problem_toposlope_x.c b/pfsimulator/parflow_lib/problem_toposlope_x.c
index cfbc81773..1e7e09b9d 100644
--- a/pfsimulator/parflow_lib/problem_toposlope_x.c
+++ b/pfsimulator/parflow_lib/problem_toposlope_x.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 #include "parflow_netcdf.h"
diff --git a/pfsimulator/parflow_lib/problem_toposlope_y.c b/pfsimulator/parflow_lib/problem_toposlope_y.c
index 4f6fc5c2d..4c250e45e 100644
--- a/pfsimulator/parflow_lib/problem_toposlope_y.c
+++ b/pfsimulator/parflow_lib/problem_toposlope_y.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 #include "parflow_netcdf.h"
diff --git a/pfsimulator/parflow_lib/problem_wc_x.c b/pfsimulator/parflow_lib/problem_wc_x.c
new file mode 100644
index 000000000..b18338729
--- /dev/null
+++ b/pfsimulator/parflow_lib/problem_wc_x.c
@@ -0,0 +1,481 @@
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
+
+#include "parflow.h"
+#include "parflow_netcdf.h"
+
+/*--------------------------------------------------------------------------
+ * Structures
+ *--------------------------------------------------------------------------*/
+
+typedef struct {
+  int type;
+  void  *data;
+  int wcx_exists;              /* check to see whether or not channel width X is used */
+} PublicXtra;
+
+typedef struct {
+  Grid   *grid3d;
+  Grid   *grid2d;
+} InstanceXtra;
+
+typedef struct {
+  NameArray regions;
+  int num_regions;
+  int     *region_indices;
+  double  *values;
+} Type0;                       /* constant regions */
+
+typedef struct {
+  char  *filename;
+  Vector *wcx_values;
+} Type1;                       /* .pfb file */
+
+typedef struct {
+  char  *filename;
+  Vector *wcx_values;
+} Type2;                       /* .nc file */
+
+/** @brief Populates region data for channel width in Y direction.
+ *
+ * @param problem_data pointer to ProblemData structure
+ * @param wc_y channel width data (in the y direction)
+ * @param dummy dummy vector
+ *
+ */
+void XChannelWidth(ProblemData *problem_data, Vector *wc_x, Vector *dummy)
+{
+  PFModule *this_module = ThisPFModule;
+  PublicXtra *public_xtra = (PublicXtra *)PFModulePublicXtra(this_module);
+  InstanceXtra *instance_xtra = (InstanceXtra *)PFModuleInstanceXtra(this_module);
+
+  Grid *grid_3d = instance_xtra->grid3d;
+
+  GrGeomSolid *gr_solid, *gr_domain;
+
+  Type0 *dummy0;
+  Type1 *dummy1;
+  Type2 *dummy2;
+
+  VectorUpdateCommHandle *handle;
+
+  SubgridArray *subgrids = GridSubgrids(grid_3d);
+
+  Subgrid *subgrid;
+  Subvector *ps_sub;
+  Subvector *wcx_values_sub;
+  double *psdat, *wcx_values_dat;
+
+  double *data;
+
+  int ix, iy, iz;
+  int nx, ny, nz;
+  int r;
+
+  int is, i, j, k, ips, ipicv;
+
+  (void)dummy;
+
+  InitVectorAll(wc_x, 0.0);
+
+  if (public_xtra->wcx_exists == 1)
+  {
+    switch ((public_xtra->type))
+    {
+      case 0: {
+        int num_regions;
+        int *region_indices;
+        double *values;
+        double value;
+        int ir;
+
+        dummy0 = (Type0 *)public_xtra->data;
+
+        num_regions = dummy0->num_regions;
+        region_indices = dummy0->region_indices;
+        values = dummy0->values;
+
+        for (ir = 0; ir < num_regions; ir++)
+        {
+          gr_solid = ProblemDataGrSolid(problem_data, region_indices[ir]);
+          value = values[ir];
+
+          ForSubgridI(is, subgrids)
+          {
+            subgrid = SubgridArraySubgrid(subgrids, is);
+            ps_sub = VectorSubvector(wc_x, is);
+
+            ix = SubgridIX(subgrid);
+            iy = SubgridIY(subgrid);
+            iz = SubgridIZ(subgrid);
+
+            nx = SubgridNX(subgrid);
+            ny = SubgridNY(subgrid);
+            nz = SubgridNZ(subgrid);
+
+            r = SubgridRX(subgrid);
+
+            data = SubvectorData(ps_sub);
+
+            GrGeomInLoop(i, j, k, gr_solid, r, ix, iy, iz, nx, ny, nz,
+            {
+              ips = SubvectorEltIndex(ps_sub, i, j, 0);
+              data[ips] = value;
+            });
+          }
+        }
+        break;
+      }
+
+      case 1: {
+        Vector *wcx_val;
+
+        dummy1 = (Type1 *)public_xtra->data;
+
+        wcx_val = dummy1->wcx_values;
+
+        gr_domain = ProblemDataGrDomain(problem_data);
+
+        ForSubgridI(is, subgrids)
+        {
+          subgrid = SubgridArraySubgrid(subgrids, is);
+          ps_sub = VectorSubvector(wc_x, is);
+          wcx_values_sub = VectorSubvector(wcx_val, is);
+
+          ix = SubgridIX(subgrid);
+          iy = SubgridIY(subgrid);
+          iz = SubgridIZ(subgrid);
+
+          nx = SubgridNX(subgrid);
+          ny = SubgridNY(subgrid);
+          nz = SubgridNZ(subgrid);
+
+          r = SubgridRX(subgrid);
+
+          psdat = SubvectorData(ps_sub);
+          wcx_values_dat = SubvectorData(wcx_values_sub);
+
+          GrGeomInLoop(i, j, k, gr_domain, r, ix, iy, iz, nx, ny, nz, {
+            ips = SubvectorEltIndex(ps_sub, i, j, 0);
+            ipicv = SubvectorEltIndex(wcx_values_sub, i, j, 0);
+
+            psdat[ips] = wcx_values_dat[ipicv];
+          })
+        }
+        break;
+      }
+
+      case 2: {
+        Vector *wcx_val;
+
+        dummy2 = (Type2 *)public_xtra->data;
+
+        wcx_val = dummy2->wcx_values;
+
+        gr_domain = ProblemDataGrDomain(problem_data);
+
+        ForSubgridI(is, subgrids)
+        {
+          subgrid = SubgridArraySubgrid(subgrids, is);
+          ps_sub = VectorSubvector(wc_x, is);
+          wcx_values_sub = VectorSubvector(wcx_val, is);
+
+          ix = SubgridIX(subgrid);
+          iy = SubgridIY(subgrid);
+          iz = SubgridIZ(subgrid);
+
+          nx = SubgridNX(subgrid);
+          ny = SubgridNY(subgrid);
+          nz = SubgridNZ(subgrid);
+
+          r = SubgridRX(subgrid);
+
+          psdat = SubvectorData(ps_sub);
+          wcx_values_dat = SubvectorData(wcx_values_sub);
+
+          GrGeomInLoop(i, j, k, gr_domain, r, ix, iy, iz, nx, ny, nz, {
+            ips = SubvectorEltIndex(ps_sub, i, j, 0);
+            ipicv = SubvectorEltIndex(wcx_values_sub, i, j, 0);
+
+            psdat[ips] = wcx_values_dat[ipicv];
+          });
+        }
+        break;
+      }
+    }
+  }
+
+  handle = InitVectorUpdate(wc_x, VectorUpdateAll);
+  FinalizeVectorUpdate(handle);
+}
+
+/** @brief Initializes InstanceXtra object for channel width Y problem
+ *
+ * @return Modified PFModule with YChannelWidth information
+ */
+PFModule *XChannelWidthInitInstanceXtra(Grid *grid3d, Grid *grid2d)
+{
+  PFModule *this_module = ThisPFModule;
+  PublicXtra *public_xtra = (PublicXtra *)PFModulePublicXtra(this_module);
+  InstanceXtra *instance_xtra;
+
+  Type1 *dummy1;
+  Type2 *dummy2;
+
+
+  if (PFModuleInstanceXtra(this_module) == NULL)
+  {
+    instance_xtra = ctalloc(InstanceXtra, 1);
+  }
+  else
+  {
+    instance_xtra = (InstanceXtra *)PFModuleInstanceXtra(this_module);
+  }
+
+  if (grid3d != NULL)
+  {
+    (instance_xtra->grid3d) = grid3d;
+  }
+
+  if (grid2d != NULL)
+  {
+    (instance_xtra->grid2d) = grid2d;
+
+    if (public_xtra->wcx_exists == 1)
+    {
+      if (public_xtra->type == 1)
+      {
+        dummy1 = (Type1 *)(public_xtra->data);
+
+        dummy1->wcx_values = NewVectorType(grid2d, 1, 1, vector_cell_centered_2D);
+
+        ReadPFBinary((dummy1->filename), (dummy1->wcx_values));
+      }
+
+      if (public_xtra->type == 2)
+      {
+        dummy2 = (Type2 *)(public_xtra->data);
+
+        dummy2->wcx_values = NewVectorType(grid2d, 1, 1, vector_cell_centered_2D);
+
+        ReadPFNC((dummy2->filename), (dummy2->wcx_values), "wc_x", 0, 2);
+      }
+    }
+  }
+
+
+  PFModuleInstanceXtra(this_module) = instance_xtra;
+  return this_module;
+}
+
+/** @brief Frees the InstanceXtra object for channel width Y problem
+ *
+ */
+void XChannelWidthFreeInstanceXtra()
+{
+  PFModule *this_module = ThisPFModule;
+  PublicXtra *public_xtra = (PublicXtra *)PFModulePublicXtra(this_module);
+  InstanceXtra *instance_xtra = (InstanceXtra *)PFModuleInstanceXtra(this_module);
+
+  Type1 *dummy1;
+  Type2 *dummy2;
+
+  if (public_xtra->wcx_exists == 1)
+  {
+    if (public_xtra->type == 1)
+    {
+      dummy1 = (Type1 *)(public_xtra->data);
+      FreeVector(dummy1->wcx_values);
+    }
+
+    if (public_xtra->type == 2)
+    {
+      dummy2 = (Type2 *)(public_xtra->data);
+      FreeVector(dummy2->wcx_values);
+    }
+  }
+
+  if (instance_xtra)
+  {
+    free(instance_xtra);
+  }
+}
+
+/** @brief Creates PublicXtra object for channel width
+ *
+ */
+PFModule *XChannelWidthNewPublicXtra()
+{
+  PFModule *this_module = ThisPFModule;
+  PublicXtra *public_xtra;
+
+  Type0 *dummy0;
+  Type1 *dummy1;
+  Type2 *dummy2;
+
+  int num_regions, ir;
+
+  char key[IDB_MAX_KEY_LEN];
+
+  char *name;
+
+  char *switch_name;
+  char *switch_exist_name;
+
+  NameArray type_na;
+  NameArray switch_na;
+
+  int switch_val;
+
+  public_xtra = ctalloc(PublicXtra, 1);
+
+  name = "Solver.Nonlinear.ChannelWidthExistX";
+  switch_na = NA_NewNameArray("False True");
+  switch_exist_name = GetStringDefault(name, "False");
+  switch_val = NA_NameToIndexExitOnError(switch_na, switch_exist_name, name);
+  NA_FreeNameArray(switch_na);
+
+  public_xtra->wcx_exists = switch_val;
+
+  if (public_xtra->wcx_exists == 1)
+  {
+    type_na = NA_NewNameArray("Constant PFBFile NCFile");
+
+    switch_name = GetString("ChannelWidthX.Type");
+
+    public_xtra->type = NA_NameToIndexExitOnError(type_na, switch_name, "ChannelWidthX.Type");
+
+    switch ((public_xtra->type))
+    {
+      case 0: {
+        dummy0 = ctalloc(Type0, 1);
+
+        switch_name = GetString("ChannelWidthX.GeomNames");
+
+        dummy0->regions = NA_NewNameArray(switch_name);
+
+        num_regions = (dummy0->num_regions) = NA_Sizeof(dummy0->regions);
+
+        (dummy0->region_indices) = ctalloc(int, num_regions);
+        (dummy0->values) = ctalloc(double, num_regions);
+
+        for (ir = 0; ir < num_regions; ir++)
+        {
+          (dummy0->region_indices)[ir] = NA_NameToIndex(GlobalsGeomNames, NA_IndexToName((dummy0->regions), ir));
+          sprintf(key, "ChannelWidthX.Geom.%s.Value", NA_IndexToName((dummy0->regions), ir));
+          (dummy0->values)[ir] = GetDouble(key);
+        }
+
+        (public_xtra->data) = (void *)dummy0;
+
+        break;
+      }
+
+      case 1: {
+        dummy1 = ctalloc(Type1, 1);
+
+        dummy1->filename = GetString("ChannelWidthX.FileName");
+
+        (public_xtra->data) = (void *)dummy1;
+        break;
+      }
+
+      case 2: {
+        dummy2 = ctalloc(Type2, 1);
+
+        dummy2->filename = GetString("ChannelWidthX.FileName");
+
+        (public_xtra->data) = (void *)dummy2;
+        break;
+      }
+
+      default: {
+        InputError("Error: invalid type <%s> for key <%s>\n", switch_name, key);
+      }
+    }
+
+    NA_FreeNameArray(type_na);
+  }
+
+  PFModulePublicXtra(this_module) = public_xtra;
+  return this_module;
+}
+
+/** @brief Frees the PublicXtra object associated with XChannelWidth.
+ *
+ */
+void XChannelWidthFreePublicXtra()
+{
+  PFModule *this_module = ThisPFModule;
+  PublicXtra *public_xtra = (PublicXtra *)PFModulePublicXtra(this_module);
+
+  Type0 *dummy0;
+
+  if (public_xtra)
+  {
+    if (public_xtra->wcx_exists == 1)
+    {
+      switch ((public_xtra->type))
+      {
+        case 0: {
+          dummy0 = (Type0 *)(public_xtra->data);
+
+          NA_FreeNameArray(dummy0->regions);
+
+          tfree(dummy0->region_indices);
+          tfree(dummy0->values);
+          tfree(dummy0);
+          break;
+        }
+
+        case 1: {
+          Type1 *dummy1;
+          dummy1 = (Type1 *)(public_xtra->data);
+
+          tfree(dummy1);
+          break;
+        }
+
+        case 2: {
+          Type2 *dummy2;
+          dummy2 = (Type2 *)(public_xtra->data);
+
+          tfree(dummy2);
+          break;
+        }
+      }
+    }
+    tfree(public_xtra);
+  }
+}
+
+
+int XChannelWidthSizeOfTempData()
+{
+  return 0;
+}
diff --git a/pfsimulator/parflow_lib/problem_wc_y.c b/pfsimulator/parflow_lib/problem_wc_y.c
new file mode 100644
index 000000000..c58895914
--- /dev/null
+++ b/pfsimulator/parflow_lib/problem_wc_y.c
@@ -0,0 +1,483 @@
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
+
+#include "parflow.h"
+#include "parflow_netcdf.h"
+
+
+/*--------------------------------------------------------------------------
+ * Structures
+ *--------------------------------------------------------------------------*/
+
+typedef struct {
+  int type;
+  void  *data;
+  int wcy_exists;              /* check to see whether or not channel width Y is used */
+} PublicXtra;
+
+typedef struct {
+  Grid   *grid3d;
+  Grid   *grid2d;
+} InstanceXtra;
+
+typedef struct {
+  NameArray regions;
+  int num_regions;
+  int     *region_indices;
+  double  *values;
+} Type0;                       /* constant regions */
+
+typedef struct {
+  char  *filename;
+  Vector *wcy_values;
+} Type1;                       /* .pfb file */
+
+typedef struct {
+  char  *filename;
+  Vector *wcy_values;
+} Type2;                       /* .nc file */
+
+
+/** @brief Populates region data for channel width in Y direction.
+ *
+ * @param problem_data pointer to ProblemData structure
+ * @param wc_y channel width data (in the y direction)
+ * @param dummy dummy vector
+ *
+ */
+void YChannelWidth(ProblemData *problem_data, Vector *wc_y, Vector *dummy)
+{
+  PFModule *this_module = ThisPFModule;
+  PublicXtra *public_xtra = (PublicXtra *)PFModulePublicXtra(this_module);
+  InstanceXtra *instance_xtra = (InstanceXtra *)PFModuleInstanceXtra(this_module);
+
+  Grid *grid_3d = instance_xtra->grid3d;
+
+  GrGeomSolid *gr_solid, *gr_domain;
+
+  Type0 *dummy0;
+  Type1 *dummy1;
+  Type2 *dummy2;
+
+  VectorUpdateCommHandle *handle;
+
+  SubgridArray *subgrids = GridSubgrids(grid_3d);
+
+  Subgrid *subgrid;
+  Subvector *ps_sub;
+  Subvector *wcy_values_sub;
+  double *psdat, *wcy_values_dat;
+
+  double *data;
+
+  int ix, iy, iz;
+  int nx, ny, nz;
+  int r;
+
+  int is, i, j, k, ips, ipicv;
+
+  (void)dummy;
+
+  InitVectorAll(wc_y, 0.0);
+
+  if (public_xtra->wcy_exists == 1)
+  {
+    switch ((public_xtra->type))
+    {
+      case 0: {
+        int num_regions;
+        int *region_indices;
+        double *values;
+        double value;
+        int ir;
+
+        dummy0 = (Type0 *)public_xtra->data;
+
+        num_regions = dummy0->num_regions;
+        region_indices = dummy0->region_indices;
+        values = dummy0->values;
+
+        for (ir = 0; ir < num_regions; ir++)
+        {
+          gr_solid = ProblemDataGrSolid(problem_data, region_indices[ir]);
+          value = values[ir];
+
+          ForSubgridI(is, subgrids)
+          {
+            subgrid = SubgridArraySubgrid(subgrids, is);
+            ps_sub = VectorSubvector(wc_y, is);
+
+            ix = SubgridIX(subgrid);
+            iy = SubgridIY(subgrid);
+            iz = SubgridIZ(subgrid);
+
+            nx = SubgridNX(subgrid);
+            ny = SubgridNY(subgrid);
+            nz = SubgridNZ(subgrid);
+
+            r = SubgridRX(subgrid);
+
+            data = SubvectorData(ps_sub);
+
+            GrGeomInLoop(i, j, k, gr_solid, r, ix, iy, iz, nx, ny, nz,
+            {
+              ips = SubvectorEltIndex(ps_sub, i, j, 0);
+              data[ips] = value;
+            });
+          }
+        }
+        break;
+      }
+
+      case 1: {
+        Vector *wcy_val;
+
+        dummy1 = (Type1 *)public_xtra->data;
+
+        wcy_val = dummy1->wcy_values;
+
+        gr_domain = ProblemDataGrDomain(problem_data);
+
+        ForSubgridI(is, subgrids)
+        {
+          subgrid = SubgridArraySubgrid(subgrids, is);
+          ps_sub = VectorSubvector(wc_y, is);
+          wcy_values_sub = VectorSubvector(wcy_val, is);
+
+          ix = SubgridIX(subgrid);
+          iy = SubgridIY(subgrid);
+          iz = SubgridIZ(subgrid);
+
+          nx = SubgridNX(subgrid);
+          ny = SubgridNY(subgrid);
+          nz = SubgridNZ(subgrid);
+
+          r = SubgridRX(subgrid);
+
+          psdat = SubvectorData(ps_sub);
+          wcy_values_dat = SubvectorData(wcy_values_sub);
+
+          GrGeomInLoop(i, j, k, gr_domain, r, ix, iy, iz, nx, ny, nz, {
+            ips = SubvectorEltIndex(ps_sub, i, j, 0);
+            ipicv = SubvectorEltIndex(wcy_values_sub, i, j, 0);
+
+            psdat[ips] = wcy_values_dat[ipicv];
+          })
+        }
+        break;
+      }
+
+      case 2: {
+        Vector *wcy_val;
+
+        dummy2 = (Type2 *)public_xtra->data;
+
+        wcy_val = dummy2->wcy_values;
+
+        gr_domain = ProblemDataGrDomain(problem_data);
+
+        ForSubgridI(is, subgrids)
+        {
+          subgrid = SubgridArraySubgrid(subgrids, is);
+          ps_sub = VectorSubvector(wc_y, is);
+          wcy_values_sub = VectorSubvector(wcy_val, is);
+
+          ix = SubgridIX(subgrid);
+          iy = SubgridIY(subgrid);
+          iz = SubgridIZ(subgrid);
+
+          nx = SubgridNX(subgrid);
+          ny = SubgridNY(subgrid);
+          nz = SubgridNZ(subgrid);
+
+          r = SubgridRX(subgrid);
+
+          psdat = SubvectorData(ps_sub);
+          wcy_values_dat = SubvectorData(wcy_values_sub);
+
+          GrGeomInLoop(i, j, k, gr_domain, r, ix, iy, iz, nx, ny, nz, {
+            ips = SubvectorEltIndex(ps_sub, i, j, 0);
+            ipicv = SubvectorEltIndex(wcy_values_sub, i, j, 0);
+
+            psdat[ips] = wcy_values_dat[ipicv];
+          });
+        }
+        break;
+      }
+    }
+  }
+
+  handle = InitVectorUpdate(wc_y, VectorUpdateAll);
+  FinalizeVectorUpdate(handle);
+}
+
+
+/** @brief Initializes InstanceXtra object for channel width Y problem
+ *
+ * @return Modified PFModule with YChannelWidth information
+ */
+PFModule *YChannelWidthInitInstanceXtra(Grid *grid3d, Grid *grid2d)
+{
+  PFModule *this_module = ThisPFModule;
+  PublicXtra *public_xtra = (PublicXtra *)PFModulePublicXtra(this_module);
+  InstanceXtra *instance_xtra;
+
+  Type1 *dummy1;
+  Type2 *dummy2;
+
+  if (PFModuleInstanceXtra(this_module) == NULL)
+  {
+    instance_xtra = ctalloc(InstanceXtra, 1);
+  }
+  else
+  {
+    instance_xtra = (InstanceXtra *)PFModuleInstanceXtra(this_module);
+  }
+
+  if (grid3d != NULL)
+  {
+    (instance_xtra->grid3d) = grid3d;
+  }
+
+  if (grid2d != NULL)
+  {
+    (instance_xtra->grid2d) = grid2d;
+
+    if (public_xtra->wcy_exists)
+    {
+      if (public_xtra->type == 1)
+      {
+        dummy1 = (Type1 *)(public_xtra->data);
+
+        dummy1->wcy_values = NewVectorType(grid2d, 1, 1, vector_cell_centered_2D);
+
+        ReadPFBinary((dummy1->filename), (dummy1->wcy_values));
+      }
+
+      if (public_xtra->type == 2)
+      {
+        dummy2 = (Type2 *)(public_xtra->data);
+
+        dummy2->wcy_values = NewVectorType(grid2d, 1, 1, vector_cell_centered_2D);
+
+        ReadPFNC((dummy2->filename), (dummy2->wcy_values), "wc_y", 0, 2);
+      }
+    }
+  }
+
+
+  PFModuleInstanceXtra(this_module) = instance_xtra;
+  return this_module;
+}
+
+/** @brief Frees the InstanceXtra object for channel width Y problem
+ *
+ */
+void YChannelWidthFreeInstanceXtra()
+{
+  PFModule *this_module = ThisPFModule;
+  PublicXtra *public_xtra = (PublicXtra *)PFModulePublicXtra(this_module);
+  InstanceXtra *instance_xtra = (InstanceXtra *)PFModuleInstanceXtra(this_module);
+
+  Type1 *dummy1;
+  Type2 *dummy2;
+
+  if (public_xtra->wcy_exists == 1)
+  {
+    if (public_xtra->type == 1)
+    {
+      dummy1 = (Type1 *)(public_xtra->data);
+      FreeVector(dummy1->wcy_values);
+    }
+
+    if (public_xtra->type == 2)
+    {
+      dummy2 = (Type2 *)(public_xtra->data);
+      FreeVector(dummy2->wcy_values);
+    }
+  }
+
+  if (instance_xtra)
+  {
+    free(instance_xtra);
+  }
+}
+
+/** @brief Creates PublicXtra object for channel width
+ *
+ */
+PFModule *YChannelWidthNewPublicXtra()
+{
+  PFModule *this_module = ThisPFModule;
+  PublicXtra *public_xtra;
+
+  Type0 *dummy0;
+  Type1 *dummy1;
+  Type2 *dummy2;
+
+  int num_regions, ir;
+
+  char key[IDB_MAX_KEY_LEN];
+
+  char *name;
+
+  char *switch_name;
+  char *switch_exist_name;
+
+  NameArray type_na;
+  NameArray switch_na;
+  int switch_val;
+
+  public_xtra = ctalloc(PublicXtra, 1);
+
+  name = "Solver.Nonlinear.ChannelWidthExistY";
+  switch_na = NA_NewNameArray("False True");
+  switch_exist_name = GetStringDefault(name, "False");
+  switch_val = NA_NameToIndexExitOnError(switch_na, switch_exist_name, name);
+  NA_FreeNameArray(switch_na);
+
+  public_xtra->wcy_exists = switch_val;
+
+  if (public_xtra->wcy_exists == 1)
+  {
+    type_na = NA_NewNameArray("Constant PFBFile NCFile");
+
+    switch_name = GetString("ChannelWidthY.Type");
+
+    public_xtra->type = NA_NameToIndexExitOnError(type_na, switch_name, "ChannelWidthY.Type");
+
+    switch ((public_xtra->type))
+    {
+      case 0: {
+        dummy0 = ctalloc(Type0, 1);
+
+        switch_name = GetString("ChannelWidthY.GeomNames");
+
+        dummy0->regions = NA_NewNameArray(switch_name);
+
+        num_regions = (dummy0->num_regions) = NA_Sizeof(dummy0->regions);
+
+        (dummy0->region_indices) = ctalloc(int, num_regions);
+        (dummy0->values) = ctalloc(double, num_regions);
+
+        for (ir = 0; ir < num_regions; ir++)
+        {
+          (dummy0->region_indices)[ir] = NA_NameToIndex(GlobalsGeomNames, NA_IndexToName((dummy0->regions), ir));
+          sprintf(key, "ChannelWidthY.Geom.%s.Value", NA_IndexToName((dummy0->regions), ir));
+          (dummy0->values)[ir] = GetDouble(key);
+        }
+
+        (public_xtra->data) = (void *)dummy0;
+
+        break;
+      }
+
+      case 1: {
+        dummy1 = ctalloc(Type1, 1);
+
+        dummy1->filename = GetString("ChannelWidthY.FileName");
+
+        (public_xtra->data) = (void *)dummy1;
+        break;
+      }
+
+      case 2: {
+        dummy2 = ctalloc(Type2, 1);
+
+        dummy2->filename = GetString("ChannelWidthY.FileName");
+
+        (public_xtra->data) = (void *)dummy2;
+        break;
+      }
+
+      default: {
+        InputError("Error: invalid type <%s> for key <%s>\n", switch_name, key);
+      }
+    }
+
+    NA_FreeNameArray(type_na);
+  }
+
+  PFModulePublicXtra(this_module) = public_xtra;
+  return this_module;
+}
+
+/** @brief Frees the PublicXtra object associated with YChannelWidth.
+ *
+ */
+void YChannelWidthFreePublicXtra()
+{
+  PFModule *this_module = ThisPFModule;
+  PublicXtra *public_xtra = (PublicXtra *)PFModulePublicXtra(this_module);
+
+  Type0 *dummy0;
+
+  if (public_xtra)
+  {
+    if (public_xtra->wcy_exists)
+    {
+      switch ((public_xtra->type))
+      {
+        case 0: {
+          dummy0 = (Type0 *)(public_xtra->data);
+
+          NA_FreeNameArray(dummy0->regions);
+
+          tfree(dummy0->region_indices);
+          tfree(dummy0->values);
+          tfree(dummy0);
+          break;
+        }
+
+        case 1: {
+          Type1 *dummy1;
+          dummy1 = (Type1 *)(public_xtra->data);
+
+          tfree(dummy1);
+          break;
+        }
+
+        case 2: {
+          Type2 *dummy2;
+          dummy2 = (Type2 *)(public_xtra->data);
+
+          tfree(dummy2);
+          break;
+        }
+      }
+    }
+
+    tfree(public_xtra);
+  }
+}
+
+
+int YChannelWidthSizeOfTempData()
+{
+  return 0;
+}
diff --git a/pfsimulator/parflow_lib/process_grid.c b/pfsimulator/parflow_lib/process_grid.c
index 5820dc70b..bc47c0e51 100644
--- a/pfsimulator/parflow_lib/process_grid.c
+++ b/pfsimulator/parflow_lib/process_grid.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routines for reading grid used to specify how the problem is
@@ -41,7 +41,7 @@
 
 Subgrid    *ReadProcessSubgrid(int subgrid_num)
 {
-  char base_name[IDB_MAX_KEY_LEN-4];
+  char base_name[IDB_MAX_KEY_LEN - 4];
   char key_name[IDB_MAX_KEY_LEN];
 
   sprintf(base_name, "ProcessGrid.%i", subgrid_num);
diff --git a/pfsimulator/parflow_lib/random.c b/pfsimulator/parflow_lib/random.c
index 7e3ef816a..55ec794f4 100644
--- a/pfsimulator/parflow_lib/random.c
+++ b/pfsimulator/parflow_lib/random.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * SeedRand, Rand
 *
diff --git a/pfsimulator/parflow_lib/rb_GS_point.c b/pfsimulator/parflow_lib/rb_GS_point.c
index ed7607173..0be1df1ea 100644
--- a/pfsimulator/parflow_lib/rb_GS_point.c
+++ b/pfsimulator/parflow_lib/rb_GS_point.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Pointwise red/black Gauss-Seidel
@@ -59,7 +59,7 @@ void     RedBlackGSPoint(
                          double  tol,
                          int     zero)
 {
-  PUSH_NVTX("RedBlackGSPoint",5)
+  PUSH_NVTX("RedBlackGSPoint", 5)
 
   PFModule       *this_module = ThisPFModule;
   PublicXtra     *public_xtra = (PublicXtra*)PFModulePublicXtra(this_module);
@@ -338,7 +338,7 @@ void     RedBlackGSPoint(
           bp = SubvectorElt(b_sub, ix, iy, iz);
 
           iv = im = 0;
-          
+
           BoxLoopI2(i, j, k, ix, iy, iz, nx, ny, nz,
                     iv, nx_v, ny_v, nz_v, sx, sy, sz,
                     im, nx_m, ny_m, nz_m, sx, sy, sz,
@@ -349,8 +349,8 @@ void     RedBlackGSPoint(
                                 a4[im] * x4[iv] +
                                 a5[im] * x5[iv] +
                                 a6[im] * x6[iv])) / a0[im];
-                                
-            SKIP_PARALLEL_SYNC;                                
+
+            SKIP_PARALLEL_SYNC;
           });
         }
       }
@@ -368,9 +368,9 @@ void     RedBlackGSPoint(
   else
     IncFLOPCount(13 * (iter * VectorSize(x)));
 
-    PARALLEL_SYNC;
+  PARALLEL_SYNC;
 
-    POP_NVTX
+  POP_NVTX
 }
 
 
diff --git a/pfsimulator/parflow_lib/read_parflow_binary.c b/pfsimulator/parflow_lib/read_parflow_binary.c
index a7e983236..a46e7e412 100644
--- a/pfsimulator/parflow_lib/read_parflow_binary.c
+++ b/pfsimulator/parflow_lib/read_parflow_binary.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routines to read a Vector from a distributed file.
diff --git a/pfsimulator/parflow_lib/read_parflow_netcdf.c b/pfsimulator/parflow_lib/read_parflow_netcdf.c
index c4152dd49..49a15bee2 100644
--- a/pfsimulator/parflow_lib/read_parflow_netcdf.c
+++ b/pfsimulator/parflow_lib/read_parflow_netcdf.c
@@ -1,25 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routines to read a Vector from a distributed file.
@@ -41,22 +46,24 @@ void ReadPFNC(char *fileName, Vector *v, char *varName, int tStep, int dimension
   SubgridArray   *subgrids = GridSubgrids(grid);
   Subgrid        *subgrid;
   Subvector      *subvector;
-  
+
   int g;
-  
-  int ncRID, varID;
-  
+
+  int ncRID;
+
+  BeginTiming(NetcdfTimingIndex);
+
   OpenNCFile(fileName, &ncRID);
 
-  nc_inq_varid(ncRID, varName, &varID);
-  
   ForSubgridI(g, subgrids)
   {
     subgrid = SubgridArraySubgrid(subgrids, g);
     subvector = VectorSubvector(v, g);
-    ReadNCFile(ncRID, varID, subvector, subgrid, varName, tStep, dimensionality);
+    ReadNCFile(fileName, ncRID, subvector, subgrid, varName, tStep, dimensionality);
   }
   nc_close(ncRID);
+
+  EndTiming(NetcdfTimingIndex);
 }
 
 void OpenNCFile(char *file_name, int *ncRID)
@@ -64,7 +71,7 @@ void OpenNCFile(char *file_name, int *ncRID)
   char *switch_name;
   char key[IDB_MAX_KEY_LEN];
   char *default_val = "None";
-  
+
   sprintf(key, "NetCDF.ROMIOhints");
   switch_name = GetStringDefault(key, "None");
   if (strcmp(switch_name, default_val) != 0)
@@ -85,7 +92,7 @@ void OpenNCFile(char *file_name, int *ncRID)
       MPI_Info_set(romio_info, romio_key, value);
     }
     int res = nc_open_par(file_name, NC_MPIIO, amps_CommWorld, romio_info, ncRID);
-    if(res != NC_NOERR)
+    if (res != NC_NOERR)
     {
       amps_Printf("Error: nc_open_par failed for file <%s>\n", file_name);
     }
@@ -93,7 +100,7 @@ void OpenNCFile(char *file_name, int *ncRID)
   else
   {
     int res = nc_open_par(file_name, NC_MPIIO, amps_CommWorld, MPI_INFO_NULL, ncRID);
-    if(res != NC_NOERR)
+    if (res != NC_NOERR)
     {
       amps_Printf("Error: nc_open_par failed for file <%s>\n", file_name);
     }
@@ -103,16 +110,31 @@ void OpenNCFile(char *file_name, int *ncRID)
 #endif
 }
 
-void ReadNCFile(int ncRID, int varID, Subvector *subvector, Subgrid *subgrid, char *varName, int tStep, int dimensionality)
+void ReadNCFile(char* filename, int ncRID, Subvector *subvector, Subgrid *subgrid, char *varName, int tStep, int dimensionality)
 {
 #ifdef PARFLOW_HAVE_NETCDF
+  int varID;
   nc_inq_varid(ncRID, varName, &varID);
 
-  if (dimensionality == 3)
+  int time_index = tStep;
+
+  if (tStep < 0)
   {
-    nc_inq_varid(ncRID, varName, &varID);
+    unsigned long end[MAX_NC_VARS];
+    find_variable_length(ncRID, varID, end);
 
+    time_index = end[0] + tStep;
 
+    if (time_index < 0)
+    {
+      char buffer[2048];
+      snprintf(buffer, 2048, "Negative index provided (%d) into NetCDF file (%s) is larger than number of timesteps present in file", time_index, filename);
+      PARFLOW_ERROR(buffer);
+    }
+  }
+
+  if (dimensionality == 3)
+  {
     int ix = SubgridIX(subgrid);
     int iy = SubgridIY(subgrid);
     int iz = SubgridIZ(subgrid);
@@ -135,7 +157,7 @@ void ReadNCFile(int ncRID, int varID, Subvector *subvector, Subgrid *subgrid, ch
     nc_data = (double*)malloc(sizeof(double) * nx * ny * nz);
 
     (void)subgrid;
-    startp[0] = tStep, countp[0] = 1;
+    startp[0] = time_index, countp[0] = 1;
     startp[1] = iz, countp[1] = nz;
     startp[2] = iy, countp[2] = ny;
     startp[3] = ix, countp[3] = nx;
@@ -167,9 +189,6 @@ void ReadNCFile(int ncRID, int varID, Subvector *subvector, Subgrid *subgrid, ch
   }
   else if (dimensionality == 2)
   {
-    nc_inq_varid(ncRID, varName, &varID);
-
-
     int ix = SubgridIX(subgrid);
     int iy = SubgridIY(subgrid);
     int iz = SubgridIZ(subgrid);
@@ -191,7 +210,7 @@ void ReadNCFile(int ncRID, int varID, Subvector *subvector, Subgrid *subgrid, ch
     double *nc_data;
     nc_data = (double*)malloc(sizeof(double) * nx * ny * nz);
 
-    startp[0] = tStep, countp[0] = 1;
+    startp[0] = time_index, countp[0] = 1;
     startp[1] = iy, countp[1] = ny;
     startp[2] = ix, countp[2] = nx;
 
@@ -220,6 +239,10 @@ void ReadNCFile(int ncRID, int varID, Subvector *subvector, Subgrid *subgrid, ch
 
     free(nc_data);
   }
+  else
+  {
+    PARFLOW_ERROR("Invalid dimension in ReadNCFile");
+  }
 #else
   amps_Printf("Parflow not compiled with NetCDF, can't read NetCDF file\n");
 #endif
diff --git a/pfsimulator/parflow_lib/reg_from_stenc.c b/pfsimulator/parflow_lib/reg_from_stenc.c
index 4c72fc359..c9b3de999 100644
--- a/pfsimulator/parflow_lib/reg_from_stenc.c
+++ b/pfsimulator/parflow_lib/reg_from_stenc.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routines for determining send and receive regions from stencil patterns.
diff --git a/pfsimulator/parflow_lib/region.c b/pfsimulator/parflow_lib/region.c
index 9ff0dbf02..13419045e 100644
--- a/pfsimulator/parflow_lib/region.c
+++ b/pfsimulator/parflow_lib/region.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Member functions for Region class.
diff --git a/pfsimulator/parflow_lib/region.h b/pfsimulator/parflow_lib/region.h
index b6e24206c..64d001145 100644
--- a/pfsimulator/parflow_lib/region.h
+++ b/pfsimulator/parflow_lib/region.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Header info for the Region structures
@@ -134,9 +134,9 @@ typedef struct {
  *--------------------------------------------------------------------------*/
 
 #define SubregionArraySubregions(subregion_array) \
-  ((subregion_array)->subregions)
+        ((subregion_array)->subregions)
 #define SubregionArraySubregion(subregion_array, i) \
-  ((subregion_array)->subregions[(i)])
+        ((subregion_array)->subregions[(i)])
 #define SubregionArraySize(subregion_array)  ((subregion_array)->size)
 
 /*--------------------------------------------------------------------------
@@ -152,60 +152,60 @@ typedef struct {
  *--------------------------------------------------------------------------*/
 
 #define RealSpaceDX(rx) \
-  (BackgroundDX(GlobalsBackground) / pow(2.0, (double)rx))
+        (BackgroundDX(GlobalsBackground) / pow(2.0, (double)rx))
 #define RealSpaceDY(ry) \
-  (BackgroundDY(GlobalsBackground) / pow(2.0, (double)ry))
+        (BackgroundDY(GlobalsBackground) / pow(2.0, (double)ry))
 #define RealSpaceDZ(rz) \
-  (BackgroundDZ(GlobalsBackground) / pow(2.0, (double)rz))
+        (BackgroundDZ(GlobalsBackground) / pow(2.0, (double)rz))
 
 #define RealSpaceX(ix, rx) \
-  (BackgroundX(GlobalsBackground) + (ix + 0.5) * RealSpaceDX(rx))
+        (BackgroundX(GlobalsBackground) + (ix + 0.5) * RealSpaceDX(rx))
 #define RealSpaceY(iy, ry) \
-  (BackgroundY(GlobalsBackground) + (iy + 0.5) * RealSpaceDY(ry))
+        (BackgroundY(GlobalsBackground) + (iy + 0.5) * RealSpaceDY(ry))
 #define RealSpaceZ(iz, rz) \
-  (BackgroundZ(GlobalsBackground) + (iz + 0.5) * RealSpaceDZ(rz))
+        (BackgroundZ(GlobalsBackground) + (iz + 0.5) * RealSpaceDZ(rz))
 
 #define IndexSpaceNX(rx) \
-  (BackgroundNX(GlobalsBackground) * (int)pow(2.0, rx))
+        (BackgroundNX(GlobalsBackground) * (int)pow(2.0, rx))
 #define IndexSpaceNY(ry) \
-  (BackgroundNY(GlobalsBackground) * (int)pow(2.0, ry))
+        (BackgroundNY(GlobalsBackground) * (int)pow(2.0, ry))
 #define IndexSpaceNZ(rz) \
-  (BackgroundNZ(GlobalsBackground) * (int)pow(2.0, rz))
+        (BackgroundNZ(GlobalsBackground) * (int)pow(2.0, rz))
 
 #define IndexSpaceX(x, rx) \
-  (pfround((x - RealSpaceX(0, rx)) / RealSpaceDX(rx)))
+        (pfround((x - RealSpaceX(0, rx)) / RealSpaceDX(rx)))
 #define IndexSpaceY(y, ry) \
-  (pfround((y - RealSpaceY(0, ry)) / RealSpaceDY(ry)))
+        (pfround((y - RealSpaceY(0, ry)) / RealSpaceDY(ry)))
 #define IndexSpaceZ(z, rz) \
-  (pfround((z - RealSpaceZ(0, rz)) / RealSpaceDZ(rz)))
+        (pfround((z - RealSpaceZ(0, rz)) / RealSpaceDZ(rz)))
 
 /*--------------------------------------------------------------------------
  * Utility macros:
  *--------------------------------------------------------------------------*/
 
 #define SubregionDX(subregion) \
-  (SubregionSX(subregion) * (RealSpaceDX(SubregionRX(subregion))))
+        (SubregionSX(subregion) * (RealSpaceDX(SubregionRX(subregion))))
 #define SubregionDY(subregion) \
-  (SubregionSY(subregion) * (RealSpaceDY(SubregionRY(subregion))))
+        (SubregionSY(subregion) * (RealSpaceDY(SubregionRY(subregion))))
 #define SubregionDZ(subregion) \
-  (SubregionSZ(subregion) * (RealSpaceDZ(SubregionRZ(subregion))))
+        (SubregionSZ(subregion) * (RealSpaceDZ(SubregionRZ(subregion))))
 
 #define SubregionX(subregion) \
-  RealSpaceX(SubregionIX(subregion), SubregionRX(subregion))
+        RealSpaceX(SubregionIX(subregion), SubregionRX(subregion))
 #define SubregionY(subregion) \
-  RealSpaceY(SubregionIY(subregion), SubregionRY(subregion))
+        RealSpaceY(SubregionIY(subregion), SubregionRY(subregion))
 #define SubregionZ(subregion) \
-  RealSpaceZ(SubregionIZ(subregion), SubregionRZ(subregion))
+        RealSpaceZ(SubregionIZ(subregion), SubregionRZ(subregion))
 
 /*--------------------------------------------------------------------------
  * Looping macros:
  *--------------------------------------------------------------------------*/
 
 #define ForSubregionI(i, subregion_array) \
-  for (i = 0; i < SubregionArraySize(subregion_array); i++)
+        for (i = 0; i < SubregionArraySize(subregion_array); i++)
 
 #define ForSubregionArrayI(i, region) \
-  for (i = 0; i < RegionSize(region); i++)
+        for (i = 0; i < RegionSize(region); i++)
 
 
 #endif
diff --git a/pfsimulator/parflow_lib/reservoir.c b/pfsimulator/parflow_lib/reservoir.c
new file mode 100644
index 000000000..c8b7643ef
--- /dev/null
+++ b/pfsimulator/parflow_lib/reservoir.c
@@ -0,0 +1,184 @@
+/*BHEADER*********************************************************************
+ *
+ *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+ *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+ *  by the Parflow Team (see the CONTRIBUTORS file)
+ *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+ *
+ *  This file is part of Parflow. For details, see
+ *  http://www.llnl.gov/casc/parflow
+ *
+ *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+ *  for the GNU Lesser General Public License.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License (as published
+ *  by the Free Software Foundation) version 2.1 dated February 1999.
+ *
+ *  This program is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+ *  and conditions of the GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ *  USA
+ **********************************************************************EHEADER*/
+
+#include "parflow.h"
+#include "../amps/mpi1/amps.h"
+#include <fcntl.h>
+#include <errno.h>
+
+
+#include <string.h>
+
+
+/*****************************************************************************
+*
+* The functions in this file are for manipulating the ReservoirData structure
+*   in ProblemData and work in conjuction with the ReservoirPackage module.
+*
+* Because of the times things are called, the New function is twinky
+* (it was basically put in to be symmetric with the New/Free paradigm
+* used through out the code) and is invoked by SetProblemData.  The Alloc
+* function actually allocates the data within the sub-structures and is
+* invoked by the ReservoirPackage (which has the data needed to compute the array
+* sizes and such).  The Free is smart enough to clean up after both the New
+* and Alloc functions and is called by SetProblemData.
+*
+*****************************************************************************/
+
+
+/*--------------------------------------------------------------------------
+ * NewReservoirData
+ *--------------------------------------------------------------------------*/
+
+ReservoirData *NewReservoirData()
+{
+  ReservoirData    *reservoir_data;
+
+  reservoir_data = ctalloc(ReservoirData, 1);
+
+  ReservoirDataNumReservoirs(reservoir_data) = -1;
+
+  ReservoirDataReservoirPhysicals(reservoir_data) = NULL;
+
+
+  return reservoir_data;
+}
+
+
+/*--------------------------------------------------------------------------
+ * FreeReservoirData
+ *--------------------------------------------------------------------------*/
+
+void FreeReservoirData(
+                       ReservoirData *reservoir_data)
+{
+  ReservoirDataPhysical *reservoir_data_physical;
+  int i;
+
+  if (reservoir_data)
+  {
+    if (ReservoirDataNumReservoirs(reservoir_data) > 0)
+    {
+      for (i = 0; i < ReservoirDataNumReservoirs(reservoir_data); i++)
+      {
+        reservoir_data_physical = ReservoirDataReservoirPhysical(reservoir_data, i);
+        if (ReservoirDataPhysicalName(reservoir_data_physical))
+        {
+          tfree(ReservoirDataPhysicalName(reservoir_data_physical));
+        }
+        if (ReservoirDataPhysicalIntakeSubgrid(reservoir_data_physical))
+        {
+          FreeSubgrid(ReservoirDataPhysicalIntakeSubgrid(reservoir_data_physical));
+        }
+        if (ReservoirDataPhysicalReleaseSubgrid(reservoir_data_physical))
+        {
+          FreeSubgrid(ReservoirDataPhysicalReleaseSubgrid(reservoir_data_physical));
+        }
+        if (ReservoirDataPhysicalSecondaryIntakeSubgrid(reservoir_data_physical))
+        {
+          FreeSubgrid(ReservoirDataPhysicalSecondaryIntakeSubgrid(reservoir_data_physical));
+        }
+        tfree(reservoir_data_physical);
+      }
+      if (ReservoirDataReservoirPhysicals(reservoir_data))
+      {
+        tfree(ReservoirDataReservoirPhysicals(reservoir_data));
+      }
+    }
+    tfree(reservoir_data);
+  }
+}
+
+
+void WriteReservoirs(
+                     char *         file_prefix,
+                     Problem *      problem,
+                     ReservoirData *reservoir_data,
+                     double         time,
+                     int            write_header)
+{
+  ReservoirDataPhysical *reservoir_data_physical;
+
+
+  int reservoir;
+
+  FILE             *file;
+
+  char file_suffix[5] = "csv";
+  char filename[255];
+
+  int p;
+
+  if (ReservoirDataNumReservoirs(reservoir_data) > 0)
+  {
+    p = amps_Rank(amps_CommWorld);
+    if (write_header)
+    {
+      if (p == 0)
+      {
+        sprintf(filename, "%s.%s", file_prefix, file_suffix);
+        file = fopen(filename, "w");
+        fprintf(file, "time");
+        fprintf(file, ",name");
+        fprintf(file, ",storage");
+        fprintf(file, ",intake_amount_since_last_row");
+        fprintf(file, ",release_amount_since_last_row");
+        fprintf(file, ",release_rate");
+        fprintf(file, "\n");
+        fclose(file);
+      }
+    }
+    for (reservoir = 0; reservoir < ReservoirDataNumReservoirs(reservoir_data); reservoir++)
+    {
+      reservoir_data_physical = ReservoirDataReservoirPhysical(reservoir_data, reservoir);
+      if (p == 0)
+      {
+        sprintf(filename, "%s.%s", file_prefix, file_suffix);
+        file = fopen(filename, "a");
+        if (file == NULL)
+        {
+          amps_Printf("Error: can't open output file %s\n", filename);
+          exit(1);
+        }
+        fprintf(file, "%f", time);
+        reservoir_data_physical = ReservoirDataReservoirPhysical(reservoir_data, reservoir);
+        fprintf(file, ",%s", ReservoirDataPhysicalName(reservoir_data_physical));
+        fprintf(file, ",%f", ReservoirDataPhysicalStorage(reservoir_data_physical));
+        fprintf(file, ",%f", ReservoirDataPhysicalIntakeAmountSinceLastPrint(reservoir_data_physical));
+        fprintf(file, ",%f", ReservoirDataPhysicalReleaseAmountSinceLastPrint(reservoir_data_physical));
+        fprintf(file, ",%f", ReservoirDataPhysicalReleaseRate(reservoir_data_physical));
+
+        ReservoirDataPhysicalReleaseAmountSinceLastPrint(reservoir_data_physical) = 0;
+        ReservoirDataPhysicalIntakeAmountSinceLastPrint(reservoir_data_physical) = 0;
+        fprintf(file, "\n");
+        fclose(file);
+      }
+    }
+  }
+}
+
diff --git a/pfsimulator/parflow_lib/reservoir.h b/pfsimulator/parflow_lib/reservoir.h
new file mode 100644
index 000000000..98b896175
--- /dev/null
+++ b/pfsimulator/parflow_lib/reservoir.h
@@ -0,0 +1,209 @@
+/*BHEADER*********************************************************************
+ *
+ *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+ *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+ *  by the Parflow Team (see the CONTRIBUTORS file)
+ *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+ *
+ *  This file is part of Parflow. For details, see
+ *  http://www.llnl.gov/casc/parflow
+ *
+ *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+ *  for the GNU Lesser General Public License.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License (as published
+ *  by the Free Software Foundation) version 2.1 dated February 1999.
+ *
+ *  This program is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+ *  and conditions of the GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ *  USA
+ **********************************************************************EHEADER*/
+
+#ifndef _RESERVOIR_HEADER
+#define _RESERVOIR_HEADER
+
+
+/*----------------------------------------------------------------
+ * Reservoir Physical Values structure
+ *----------------------------------------------------------------*/
+
+typedef struct {
+  int number;
+  char          *name;
+  double intake_x_lower, intake_y_lower, z_lower;
+  double intake_x_upper, intake_y_upper, z_upper;
+  int has_secondary_intake_cell;
+  double secondary_intake_x_lower, secondary_intake_y_lower;
+  int intake_cell_mpi_rank, secondary_intake_cell_mpi_rank, release_cell_mpi_rank;
+  double secondary_intake_x_upper, secondary_intake_y_upper;
+  double release_x_lower, release_y_lower, release_z_lower;
+  double release_x_upper, release_y_upper, release_z_upper;
+  double max_storage, min_release_storage, storage, release_rate;
+  double intake_amount_since_last_print, release_amount_since_last_print, release_amount_in_solver;
+  Subgrid       *intake_subgrid;
+  Subgrid       *secondary_intake_subgrid;
+  Subgrid       *release_subgrid;
+  double size;
+    #ifdef PARFLOW_HAVE_MPI
+  MPI_Comm mpi_communicator;
+    #endif
+} ReservoirDataPhysical;
+
+/*------------------------------------------------------------------
+ * Reservoir Data structure
+ *----------------------------------------------------------------*/
+
+typedef struct {
+  int num_reservoirs;
+  int overland_flow_solver;
+  ReservoirDataPhysical  **reservoir_physicals;
+} ReservoirData;
+
+/*--------------------------------------------------------------------------
+ * Accessor macros: ReservoirDataPhysical
+ *--------------------------------------------------------------------------*/
+#define ReservoirDataPhysicalNumber(reservoir_data_physical) \
+        ((reservoir_data_physical)->number)
+
+#define ReservoirDataPhysicalName(reservoir_data_physical) \
+        ((reservoir_data_physical)->name)
+
+#define ReservoirDataPhysicalMpiCommunicator(reservoir_data_physical) \
+        ((reservoir_data_physical)->mpi_communicator)
+
+#define ReservoirDataPhysicalIntakeCellMpiRank(reservoir_data_physical) \
+        ((reservoir_data_physical)->intake_cell_mpi_rank)
+
+#define ReservoirDataPhysicalIntakeXLower(reservoir_data_physical) \
+        ((reservoir_data_physical)->intake_x_lower)
+
+#define ReservoirDataPhysicalIntakeYLower(reservoir_data_physical) \
+        ((reservoir_data_physical)->intake_y_lower)
+
+#define ReservoirDataPhysicalSecondaryIntakeCellMpiRank(reservoir_data_physical) \
+        ((reservoir_data_physical)->secondary_intake_cell_mpi_rank)
+
+#define ReservoirDataPhysicalSecondaryIntakeXLower(reservoir_data_physical) \
+        ((reservoir_data_physical)->secondary_intake_x_lower)
+
+#define ReservoirDataPhysicalSecondaryIntakeYLower(reservoir_data_physical) \
+        ((reservoir_data_physical)->secondary_intake_y_lower)
+
+
+#define ReservoirDataPhysicalIntakeXUpper(reservoir_data_physical) \
+        ((reservoir_data_physical)->intake_x_upper)
+
+#define ReservoirDataPhysicalIntakeYUpper(reservoir_data_physical) \
+        ((reservoir_data_physical)->intake_y_upper)
+
+#define ReservoirDataPhysicalSecondaryIntakeXUpper(reservoir_data_physical) \
+        ((reservoir_data_physical)->intake_x_upper)
+
+#define ReservoirDataPhysicalSecondaryIntakeYUpper(reservoir_data_physical) \
+        ((reservoir_data_physical)->intake_y_upper)
+
+
+#define ReservoirDataPhysicalIntakeSubgrid(reservoir_data_physical) \
+        ((reservoir_data_physical)->intake_subgrid)
+
+#define ReservoirDataPhysicalSecondaryIntakeSubgrid(reservoir_data_physical) \
+        ((reservoir_data_physical)->secondary_intake_subgrid)
+
+#define ReservoirDataPhysicalHasSecondaryIntakeCell(reservoir_data_physical) \
+        ((reservoir_data_physical)->has_secondary_intake_cell)
+
+#define ReservoirDataPhysicalReleaseCellMpiRank(reservoir_data_physical) \
+        ((reservoir_data_physical)->release_cell_mpi_rank)
+
+#define ReservoirDataPhysicalReleaseXLower(reservoir_data_physical) \
+        ((reservoir_data_physical)->release_x_lower)
+
+#define ReservoirDataPhysicalReleaseYLower(reservoir_data_physical) \
+        ((reservoir_data_physical)->release_y_lower)
+
+#define ReservoirDataPhysicalReleaseXUpper(reservoir_data_physical) \
+        ((reservoir_data_physical)->release_x_upper)
+
+#define ReservoirDataPhysicalReleaseYUpper(reservoir_data_physical) \
+        ((reservoir_data_physical)->release_y_upper)
+
+
+#define ReservoirDataPhysicalReleaseSubgrid(reservoir_data_physical) \
+        ((reservoir_data_physical)->release_subgrid)
+
+#define ReservoirDataPhysicalSize(reservoir_data_physical) \
+        ((reservoir_data_physical)->size)
+
+
+#define ReservoirDataPhysicalMaxStorage(reservoir_data_physical) \
+        ((reservoir_data_physical)->max_storage)
+
+#define ReservoirDataPhysicalStorage(reservoir_data_physical) \
+        ((reservoir_data_physical)->storage)
+
+#define ReservoirDataPhysicalReleaseAmountSinceLastPrint(reservoir_data_physical) \
+        ((reservoir_data_physical)->release_amount_since_last_print)
+
+#define ReservoirDataPhysicalIntakeAmountSinceLastPrint(reservoir_data_physical) \
+        ((reservoir_data_physical)->intake_amount_since_last_print)
+
+#define ReservoirDataPhysicalReleaseAmountInSolver(reservoir_data_physical) \
+        ((reservoir_data_physical)->release_amount_in_solver)
+
+#define ReservoirDataPhysicalMinReleaseStorage(reservoir_data_physical) \
+        ((reservoir_data_physical)->min_release_storage)
+
+#define ReservoirDataPhysicalReleaseRate(reservoir_data_physical) \
+        ((reservoir_data_physical)->release_rate)
+
+
+
+/*--------------------------------------------------------------------------
+ * Accessor macros: ReservoirData
+ *--------------------------------------------------------------------------*/
+
+
+
+/*---------------------------- Flux reservoir data ------------------------------*/
+#define ReservoirDataNumReservoirs(reservoir_data)      ((reservoir_data)->num_reservoirs)
+
+#define ReservoirDataOverlandFlowSolver(reservoir_data)      ((reservoir_data)->overland_flow_solver)
+
+#define ReservoirDataReservoirPhysicals(reservoir_data) \
+        ((reservoir_data)->reservoir_physicals)
+#define ReservoirDataReservoirPhysical(reservoir_data, i) \
+        ((reservoir_data)->reservoir_physicals[i])
+
+
+
+
+/*--------------------------------------------------------------------------
+ * Reservoir Data constants used in the program.
+ *--------------------------------------------------------------------------*/
+
+/*       Actions        */
+#define INJECTION_RESERVOIR  0
+#define EXTRACTION_RESERVOIR 1
+
+/*       Methods        */
+/* These should match  with */
+/* the Name Array Defs from */
+/* the  Reservoir Package input. */
+#define FLUX_STANDARD  0
+#define FLUX_WEIGHTED  1
+#define FLUX_PATTERNED 2
+
+#define OVERLAND_FLOW 0
+#define OVERLAND_KINEMATIC 1
+
+/*     Write Options      */
+#define RESERVOIRDATA_WRITEHEADER     1
+
+#endif
\ No newline at end of file
diff --git a/pfsimulator/parflow_lib/reservoir_package.c b/pfsimulator/parflow_lib/reservoir_package.c
new file mode 100644
index 000000000..e71bf334c
--- /dev/null
+++ b/pfsimulator/parflow_lib/reservoir_package.c
@@ -0,0 +1,625 @@
+/*BHEADER*********************************************************************
+ *
+ *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+ *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+ *  by the Parflow Team (see the CONTRIBUTORS file)
+ *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+ *
+ *  This file is part of Parflow. For details, see
+ *  http://www.llnl.gov/casc/parflow
+ *
+ *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+ *  for the GNU Lesser General Public License.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License (as published
+ *  by the Free Software Foundation) version 2.1 dated February 1999.
+ *
+ *  This program is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+ *  and conditions of the GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ *  USA
+ **********************************************************************EHEADER*/
+
+#include "parflow.h"
+#include <stdio.h>
+#include <string.h>
+#include <stdbool.h>
+
+/*--------------------------------------------------------------------------
+ * Structures
+ *--------------------------------------------------------------------------*/
+typedef struct {
+  /* reservoir info */
+  int num_reservoirs;
+
+  int       *type;
+  void     **data;
+
+  int overland_flow_solver;
+  NameArray reservoir_names;
+} PublicXtra;
+
+typedef void InstanceXtra;
+
+typedef struct {
+  char    *name;
+  double intake_x_location, intake_y_location;
+  int has_secondary_intake_cell;
+  double secondary_intake_x_location, secondary_intake_y_location;
+  double release_x_location, release_y_location;
+  double max_storage, min_release_storage, storage, release_rate;
+} Type0;                      /* basic vertical reservoir */
+
+/*--------------------------------------------------------------------------
+ * ReservoirPackage
+ *--------------------------------------------------------------------------*/
+
+// Temp solution while I get this fix checked in as part of it's own independent module that handles
+// var dz correctly. Works for all cases that I am aware of. (Ben West)
+/** @brief Calculates a subgrids total volume, accounting for variable dz. Assumes subgrid is fully
+ * contained within the current rank.
+ *
+ * @param subgrid the subgrid we are checking
+ * @param problem_data the problems problem data structure
+ * @return the volume of the subgrid
+ */
+double GetSubgridVolume(Subgrid *subgrid, ProblemData* problem_data)
+{
+  double dx = SubgridDX(subgrid);
+  double dy = SubgridDY(subgrid);
+  double dz = SubgridDZ(subgrid);
+  GrGeomSolid *gr_domain = problem_data->gr_domain;
+
+  double volume = 0;
+  SubgridArray   *subgrids = problem_data->dz_mult->grid->subgrids;
+  Subgrid        *tmp_subgrid;
+  int subgrid_index;
+
+  ForSubgridI(subgrid_index, subgrids)
+  {
+    tmp_subgrid = SubgridArraySubgrid(subgrids, subgrid_index);
+    Subvector *dz_mult_subvector = VectorSubvector(problem_data->dz_mult, subgrid_index);
+    double* dz_mult_data = SubvectorData(dz_mult_subvector);
+    Subgrid *intersection = IntersectSubgrids(subgrid, tmp_subgrid);
+    int nx = SubgridNX(intersection);
+    int ny = SubgridNY(intersection);
+    int nz = SubgridNZ(intersection);
+    int r = SubgridRZ(intersection);
+    int ix = SubgridIX(intersection);
+    int iy = SubgridIY(intersection);
+    int iz = SubgridIZ(intersection);
+    int i, j, k;
+    GrGeomInLoop(i, j, k, gr_domain, r, ix, iy, iz, nx, ny, nz,
+    {
+      int index = SubvectorEltIndex(dz_mult_subvector, i, j, k);
+      double dz_mult = dz_mult_data[index];
+      volume += dz_mult * dx * dy * dz;
+    });
+  }
+  return volume;
+};
+
+
+/** @brief Checks whether a subgrid intersects with the current ranks subgrid
+ *
+ * @param subgrid the subgrid we are checking
+ * @param grid the problems grid
+ * @return True or False corresponding to whether the subgrid intersects
+ */
+bool SubgridLivesOnThisRank(Subgrid* subgrid, Grid *grid)
+{
+  int subgrid_index;
+  Subgrid* rank_subgrid, *tmp_subgrid;
+
+  ForSubgridI(subgrid_index, GridSubgrids(grid))
+  {
+    rank_subgrid = SubgridArraySubgrid(GridSubgrids(grid), subgrid_index);
+    if ((tmp_subgrid = IntersectSubgrids(rank_subgrid, subgrid)))
+    {
+      return true;
+    }
+  }
+  return false;
+}
+
+/** @brief Sets the slops at the outlet faces of a cell to 0 to stop flow.
+ * Assumes an overlandkinematic boundary condition
+ *
+ *
+ *
+ * @param i the x index of the cell in question
+ * @param j the y index of the cell in question
+ * @return Null, but modifies the problem datas x and y slopes
+ */
+void StopOutletFlowAtCellOverlandKinematic(int i, int j, ProblemData* problem_data, Grid* grid)
+{
+  Vector      *slope_x = ProblemDataTSlopeX(problem_data);
+  Vector      *slope_y = ProblemDataTSlopeY(problem_data);
+  Subvector   *slope_x_subvector;
+  Subvector   *slope_y_subvector;
+  int index_slope_x;
+  int index_slope_y;
+  double *slope_x_ptr;
+  double *slope_y_ptr;
+  int subgrid_index;
+  Subgrid *subgrid;
+
+  ForSubgridI(subgrid_index, GridSubgrids(grid))
+  {
+    int subgrid_x_floor, subgrid_y_floor, subgrid_x_ceiling, subgrid_y_ceiling;
+
+    subgrid = GridSubgrid(grid, subgrid_index);
+    slope_x_subvector = VectorSubvector(slope_x, subgrid_index);
+    slope_y_subvector = VectorSubvector(slope_y, subgrid_index);
+    slope_x_ptr = SubvectorData(slope_x_subvector);
+    slope_y_ptr = SubvectorData(slope_y_subvector);
+    index_slope_y = SubvectorEltIndex(slope_y_subvector, i, j, 0);
+    subgrid_x_floor = SubgridIX(subgrid);
+    subgrid_y_floor = SubgridIY(subgrid);
+    subgrid_x_ceiling = subgrid_x_floor + SubgridNX(subgrid) - 1;
+    subgrid_y_ceiling = subgrid_y_floor + SubgridNY(subgrid) - 1;
+
+    // Check all 4 faces, as long as they live on this subgrid. First the East face
+    if (i + 1 >= subgrid_x_floor && i + 1 <= subgrid_x_ceiling && j >= subgrid_y_floor && j <= subgrid_y_ceiling)
+    {
+      index_slope_x = SubvectorEltIndex(slope_x_subvector, i + 1, j, 0);
+      if (slope_x_ptr[index_slope_x] > 0)
+      {
+        slope_x_ptr[index_slope_x] = 0;
+      }
+    }
+    // South face
+    if (i >= subgrid_x_floor && i <= subgrid_x_ceiling && j - 1 >= subgrid_y_floor && j - 1 <= subgrid_y_ceiling)
+    {
+      index_slope_y = SubvectorEltIndex(slope_y_subvector, i, j - 1, 0);
+      if (slope_y_ptr[index_slope_y] > 0)
+      {
+        slope_y_ptr[index_slope_y] = 0;
+      }
+    }
+    if (i >= subgrid_x_floor && i <= subgrid_x_ceiling && j >= subgrid_y_floor && j <= subgrid_y_ceiling)
+    {
+      index_slope_x = SubvectorEltIndex(slope_x_subvector, i, j, 0);
+      if (slope_x_ptr[index_slope_x] < 0)
+      {
+        slope_x_ptr[index_slope_x] = 0;
+      }
+    }
+    // South face
+    if (i >= subgrid_x_floor && i <= subgrid_x_ceiling && j >= subgrid_y_floor && j <= subgrid_y_ceiling)
+    {
+      index_slope_y = SubvectorEltIndex(slope_y_subvector, i, j, 0);
+      if (slope_y_ptr[index_slope_y] < 0)
+      {
+        slope_y_ptr[index_slope_y] = 0;
+      }
+    }
+  }
+}
+
+/** @brief Sets the slops at the outlet faces of a cell to 0 to stop flow.
+ * Assumes an overlandflow boundary condition
+ *
+ *
+ *
+ * @param i the x index of the cell in question
+ * @param j the y index of the cell in question
+ * @return Null, but modifies the problem datas x and y slopes
+ */
+void StopOutletFlowAtCellOverlandFlow(int i, int j, ProblemData* problem_data, Grid* grid)
+{
+  Vector      *slope_x = ProblemDataTSlopeX(problem_data);
+  Vector      *slope_y = ProblemDataTSlopeY(problem_data);
+  Subvector   *slope_x_subvector;
+  Subvector   *slope_y_subvector;
+  int index_slope_x;
+  int index_slope_y;
+  double *slope_x_ptr;
+  double *slope_y_ptr;
+  int subgrid_index;
+
+  ForSubgridI(subgrid_index, GridSubgrids(grid))
+  {
+    slope_x_subvector = VectorSubvector(slope_x, subgrid_index);
+    slope_y_subvector = VectorSubvector(slope_y, subgrid_index);
+    index_slope_y = SubvectorEltIndex(slope_y_subvector, i, j, 0);
+    index_slope_x = SubvectorEltIndex(slope_y_subvector, i, j, 0);
+    slope_x_ptr = SubvectorData(slope_x_subvector);
+    slope_y_ptr = SubvectorData(slope_y_subvector);
+    slope_x_ptr[index_slope_x] = 0;
+    slope_y_ptr[index_slope_y] = 0;
+  }
+}
+
+void         ReservoirPackage(
+                              ProblemData *problem_data)
+{
+  PFModule         *this_module = ThisPFModule;
+  PublicXtra       *public_xtra = (PublicXtra*)PFModulePublicXtra(this_module);
+
+  Type0            *dummy0;
+
+  Subgrid          *new_intake_subgrid;
+  Subgrid          *new_secondary_intake_subgrid;
+  Subgrid          *new_release_subgrid;
+
+  ReservoirData         *reservoir_data = ProblemDataReservoirData(problem_data);
+  ReservoirDataPhysical *reservoir_data_physical;
+
+  int i, reservoir_index;
+
+  int intake_ix, intake_iy;
+  int secondary_intake_ix, secondary_intake_iy;
+  int release_ix, release_iy;
+  int iz_lower;
+  int nx, ny, nz;
+  int rx, ry, rz;
+  int current_mpi_rank;
+  int intake_cell_rank, secondary_intake_cell_rank, release_cell_rank;
+  int split_color;
+  int grid_nz;
+
+  double release_subgrid_volume;
+  bool part_of_reservoir_lives_on_this_rank;
+
+  /* Allocate the reservoir data */
+
+  ReservoirDataNumReservoirs(reservoir_data) = (public_xtra->num_reservoirs);
+  //it feels like there should be a better way to get the grid than this,but I couldn't find one...
+  Grid* grid = VectorGrid(problem_data->rsz);
+  grid_nz = grid->background->nz;
+
+  if ((public_xtra->num_reservoirs) > 0)
+  {
+    ReservoirDataReservoirPhysicals(reservoir_data) = ctalloc(ReservoirDataPhysical *, (public_xtra->num_reservoirs));
+  }
+
+  reservoir_index = 0;
+
+  if ((public_xtra->num_reservoirs) > 0)
+  {
+    /* Load the reservoir data */
+    for (i = 0; i < (public_xtra->num_reservoirs); i++)
+    {
+      dummy0 = (Type0*)(public_xtra->data[i]);
+      reservoir_data_physical = ctalloc(ReservoirDataPhysical, 1);
+      intake_ix = IndexSpaceX((dummy0->intake_x_location), 0);
+      intake_iy = IndexSpaceY((dummy0->intake_y_location), 0);
+      secondary_intake_ix = IndexSpaceX((dummy0->secondary_intake_x_location), 0);
+      secondary_intake_iy = IndexSpaceY((dummy0->secondary_intake_y_location), 0);
+      release_ix = IndexSpaceX((dummy0->release_x_location), 0);
+      release_iy = IndexSpaceY((dummy0->release_y_location), 0);
+
+      secondary_intake_cell_rank = -1;
+      release_cell_rank = -1;
+
+      iz_lower = grid_nz - 1;
+
+      nx = 1;
+      ny = 1;
+      nz = 1;
+
+      rx = 0;
+      ry = 0;
+      rz = 0;
+#ifdef PARFLOW_HAVE_MPI
+      current_mpi_rank = amps_Rank(amps_CommWorld);
+#else
+      current_mpi_rank = 1;
+#endif
+
+      new_intake_subgrid = NewSubgrid(intake_ix, intake_iy, iz_lower,
+                                      nx, ny, nz,
+                                      rx, ry, rz,
+                                      current_mpi_rank);
+
+      new_secondary_intake_subgrid = NewSubgrid(secondary_intake_ix, secondary_intake_iy, iz_lower,
+                                                nx, ny, nz,
+                                                rx, ry, rz,
+                                                current_mpi_rank);
+
+      new_release_subgrid = NewSubgrid(release_ix, release_iy, iz_lower,
+                                       nx, ny, nz,
+                                       rx, ry, rz,
+                                       current_mpi_rank);
+
+      if (SubgridLivesOnThisRank(new_intake_subgrid, grid))
+      {
+        intake_cell_rank = current_mpi_rank;
+      }
+
+      if (SubgridLivesOnThisRank(new_secondary_intake_subgrid, grid))
+      {
+        secondary_intake_cell_rank = current_mpi_rank;
+      }
+      release_subgrid_volume = 1;
+      if (SubgridLivesOnThisRank(new_release_subgrid, grid))
+      {
+        release_cell_rank = current_mpi_rank;
+        release_subgrid_volume = GetSubgridVolume(new_release_subgrid, problem_data);
+      }
+      //If we are multiprocessor we need to do some reductions to determine the correct ranks
+      // for the reservoirs
+#ifdef PARFLOW_HAVE_MPI
+      amps_Invoice reservoir_cells_invoice = amps_NewInvoice("%i%i%i", &intake_cell_rank, &secondary_intake_cell_rank, &release_cell_rank);
+      amps_AllReduce(amps_CommWorld, reservoir_cells_invoice, amps_Max);
+      amps_FreeInvoice(reservoir_cells_invoice);
+
+      part_of_reservoir_lives_on_this_rank =
+        (release_cell_rank == current_mpi_rank) ||
+        (intake_cell_rank == current_mpi_rank) ||
+        (secondary_intake_cell_rank == current_mpi_rank);
+
+      MPI_Comm new_reservoir_communicator;
+      split_color = part_of_reservoir_lives_on_this_rank ? 1 : MPI_UNDEFINED;
+      MPI_Comm_split(amps_CommWorld, split_color, current_mpi_rank, &new_reservoir_communicator);
+#endif
+//     edit the slopes to prevent stuff running through the reservoir
+      if (ReservoirDataOverlandFlowSolver(reservoir_data) == OVERLAND_FLOW)
+      {
+        StopOutletFlowAtCellOverlandFlow(intake_ix, intake_iy, problem_data, grid);
+        if (ReservoirDataPhysicalHasSecondaryIntakeCell(reservoir_data_physical))
+        {
+          StopOutletFlowAtCellOverlandFlow(secondary_intake_ix, secondary_intake_iy, problem_data, grid);
+        }
+      }
+      else if (ReservoirDataOverlandFlowSolver(reservoir_data) == OVERLAND_KINEMATIC)
+      {
+        StopOutletFlowAtCellOverlandKinematic(intake_ix, intake_iy, problem_data, grid);
+        if (ReservoirDataPhysicalHasSecondaryIntakeCell(reservoir_data_physical))
+        {
+          StopOutletFlowAtCellOverlandKinematic(secondary_intake_ix, secondary_intake_iy, problem_data, grid);
+        }
+      }
+
+      reservoir_data_physical = ctalloc(ReservoirDataPhysical, 1);
+      ReservoirDataPhysicalName(reservoir_data_physical) = ctalloc(char, strlen((dummy0->name)) + 1);
+      strcpy(ReservoirDataPhysicalName(reservoir_data_physical), (dummy0->name));
+      ReservoirDataPhysicalIntakeCellMpiRank(reservoir_data_physical) = (intake_cell_rank);
+      ReservoirDataPhysicalIntakeXLower(reservoir_data_physical) = (dummy0->intake_x_location);
+      ReservoirDataPhysicalIntakeYLower(reservoir_data_physical) = (dummy0->intake_y_location);
+      ReservoirDataPhysicalSecondaryIntakeCellMpiRank(reservoir_data_physical) = (secondary_intake_cell_rank);
+      ReservoirDataPhysicalSecondaryIntakeXLower(reservoir_data_physical) = (dummy0->secondary_intake_x_location);
+      ReservoirDataPhysicalSecondaryIntakeYLower(reservoir_data_physical) = (dummy0->secondary_intake_y_location);
+      ReservoirDataPhysicalHasSecondaryIntakeCell(reservoir_data_physical) = (dummy0->has_secondary_intake_cell);
+      ReservoirDataPhysicalReleaseCellMpiRank(reservoir_data_physical) = (release_cell_rank);
+      ReservoirDataPhysicalReleaseXLower(reservoir_data_physical) = (dummy0->release_x_location);
+      ReservoirDataPhysicalReleaseYLower(reservoir_data_physical) = (dummy0->release_y_location);
+      ReservoirDataPhysicalIntakeXUpper(reservoir_data_physical) = (dummy0->intake_x_location);
+      ReservoirDataPhysicalIntakeYUpper(reservoir_data_physical) = (dummy0->intake_y_location);
+      ReservoirDataPhysicalSecondaryIntakeXUpper(reservoir_data_physical) = (dummy0->secondary_intake_x_location);
+      ReservoirDataPhysicalSecondaryIntakeYUpper(reservoir_data_physical) = (dummy0->secondary_intake_y_location);
+      ReservoirDataPhysicalReleaseXUpper(reservoir_data_physical) = (dummy0->release_x_location);
+      ReservoirDataPhysicalReleaseYUpper(reservoir_data_physical) = (dummy0->release_y_location);
+      ReservoirDataPhysicalMaxStorage(reservoir_data_physical) = (dummy0->max_storage);
+      ReservoirDataPhysicalMinReleaseStorage(reservoir_data_physical) = (dummy0->min_release_storage);
+      ReservoirDataPhysicalIntakeAmountSinceLastPrint(reservoir_data_physical) = (0);
+      ReservoirDataPhysicalReleaseAmountSinceLastPrint(reservoir_data_physical) = (0);
+      ReservoirDataPhysicalReleaseAmountInSolver(reservoir_data_physical) = (0);
+      ReservoirDataPhysicalReleaseRate(reservoir_data_physical) = (dummy0->release_rate);
+      ReservoirDataPhysicalStorage(reservoir_data_physical) = (dummy0->storage);
+      ReservoirDataPhysicalIntakeSubgrid(reservoir_data_physical) = new_intake_subgrid;
+      ReservoirDataPhysicalSecondaryIntakeSubgrid(reservoir_data_physical) = new_secondary_intake_subgrid;
+      ReservoirDataPhysicalReleaseSubgrid(reservoir_data_physical) = new_release_subgrid;
+      ReservoirDataPhysicalSize(reservoir_data_physical) = release_subgrid_volume;
+      ReservoirDataReservoirPhysical(reservoir_data, reservoir_index) = reservoir_data_physical;
+#ifdef PARFLOW_HAVE_MPI
+      reservoir_data_physical->mpi_communicator = new_reservoir_communicator;
+#endif
+      /* Put in values for this reservoir */
+      reservoir_index++;
+    }
+    ReservoirDataNumReservoirs(reservoir_data) = (reservoir_index);
+  }
+}
+
+
+/*--------------------------------------------------------------------------
+ * ReservoirPackageInitInstanceXtra
+ *--------------------------------------------------------------------------*/
+PFModule *ReservoirPackageInitInstanceXtra()
+{
+  PFModule      *this_module = ThisPFModule;
+  InstanceXtra  *instance_xtra;
+
+  instance_xtra = NULL;
+
+  PFModuleInstanceXtra(this_module) = instance_xtra;
+  return this_module;
+}
+
+
+/*--------------------------------------------------------------------------
+ * ReservoirPackageFreeInstanceXtra
+ *--------------------------------------------------------------------------*/
+
+void  ReservoirPackageFreeInstanceXtra()
+{
+  PFModule      *this_module = ThisPFModule;
+  InstanceXtra  *instance_xtra = (InstanceXtra*)PFModuleInstanceXtra(this_module);
+
+  if (instance_xtra)
+  {
+    tfree(instance_xtra);
+  }
+}
+
+
+/*--------------------------------------------------------------------------
+ * ReservoirPackageNewPublicXtra
+ *--------------------------------------------------------------------------*/
+
+PFModule  *ReservoirPackageNewPublicXtra()
+{
+  PFModule      *this_module = ThisPFModule;
+  PublicXtra    *public_xtra;
+
+  Type0         *dummy0;
+
+
+  char *reservoir_names;
+  char * EMPTY_NAMES_LIST = "";
+  char *reservoir_name;
+
+  char key[IDB_MAX_KEY_LEN];
+
+  int num_reservoirs;
+
+  char          *switch_name;
+  int switch_value;
+  NameArray overland_flow_solver_na;
+
+  public_xtra = ctalloc(PublicXtra, 1);
+
+  reservoir_names = GetStringDefault("Reservoirs.Names", EMPTY_NAMES_LIST);
+
+  public_xtra->reservoir_names = NA_NewNameArray(reservoir_names);
+
+  num_reservoirs = NA_Sizeof(public_xtra->reservoir_names);
+
+
+  if (num_reservoirs > 0)
+  {
+    (public_xtra->type) = ctalloc(int, num_reservoirs);
+    (public_xtra->data) = ctalloc(void *, num_reservoirs);
+    overland_flow_solver_na = NA_NewNameArray("OverlandFlow OverlandKinematic");
+    sprintf(key, "Reservoirs.Overland_Flow_Solver");
+    switch_name = GetString(key);
+    switch_value = NA_NameToIndexExitOnError(overland_flow_solver_na, switch_name, key);
+    switch (switch_value)
+    {
+      case 0:
+      {
+        public_xtra->overland_flow_solver = OVERLAND_FLOW;
+        break;
+      }
+
+      case 1:
+      {
+        public_xtra->overland_flow_solver = OVERLAND_KINEMATIC;
+        break;
+      }
+
+      default:
+      {
+        InputError("Reservoirs.Overland_Flow_Solver must be one of OverlandFlow or OverlandKinematic, not %s%s\n", switch_name, "");
+      }
+    }
+
+    for (int i = 0; i < num_reservoirs; i++)
+    {
+      reservoir_name = NA_IndexToName(public_xtra->reservoir_names, i);
+
+      dummy0 = ctalloc(Type0, 1);
+      dummy0->name = strdup(reservoir_name);
+
+      sprintf(key, "Reservoirs.%s.Release_X", reservoir_name);
+      dummy0->release_x_location = GetDouble(key);
+
+      sprintf(key, "Reservoirs.%s.Release_Y", reservoir_name);
+      dummy0->release_y_location = GetDouble(key);
+
+      sprintf(key, "Reservoirs.%s.Intake_X", reservoir_name);
+      dummy0->intake_x_location = GetDouble(key);
+
+      sprintf(key, "Reservoirs.%s.Intake_Y", reservoir_name);
+      dummy0->intake_y_location = GetDouble(key);
+
+      sprintf(key, "Reservoirs.%s.Has_Secondary_Intake_Cell", reservoir_name);
+      dummy0->has_secondary_intake_cell = GetInt(key);
+      // I would use a name array to do this validation, but the input is an int. Long term there
+      // needs to be a fix for bool values that are nested under a dynamic key (e.g. the reservoir name)
+      // within the table reader code
+      if (!(dummy0->has_secondary_intake_cell == 0 || dummy0->has_secondary_intake_cell == 1))
+      {
+        InputError("Reservoirs.%s.HasSecondaryIntakeCell must be one of 0 (if false) or 1 (if true)", reservoir_name, "");
+      }
+
+      if (dummy0->has_secondary_intake_cell)
+      {
+        sprintf(key, "Reservoirs.%s.Secondary_Intake_X", reservoir_name);
+        dummy0->secondary_intake_x_location = GetDouble(key);
+
+        sprintf(key, "Reservoirs.%s.Secondary_Intake_Y", reservoir_name);
+        dummy0->secondary_intake_y_location = GetDouble(key);
+      }
+      else
+      {
+        dummy0->secondary_intake_x_location = -1;
+        dummy0->secondary_intake_y_location = -1;
+      };
+      sprintf(key, "Reservoirs.%s.Min_Release_Storage", reservoir_name);
+      dummy0->min_release_storage = GetDouble(key);
+
+      sprintf(key, "Reservoirs.%s.Release_Rate", reservoir_name);
+      dummy0->release_rate = GetDouble(key);
+
+      sprintf(key, "Reservoirs.%s.Max_Storage", reservoir_name);
+      dummy0->max_storage = GetDouble(key);
+
+      sprintf(key, "Reservoirs.%s.Storage", reservoir_name);
+      dummy0->storage = GetDouble(key);
+
+      (public_xtra->num_reservoirs)++;
+      (public_xtra->data[i]) = (void*)dummy0;
+    }
+  }
+
+
+  PFModulePublicXtra(this_module) = public_xtra;
+  return this_module;
+}
+
+
+/*-------------------------------------------------------------------------
+ * ReservoirPackageFreePublicXtra
+ *-------------------------------------------------------------------------*/
+
+void  ReservoirPackageFreePublicXtra()
+{
+  PFModule    *this_module = ThisPFModule;
+  PublicXtra  *public_xtra = (PublicXtra*)PFModulePublicXtra(this_module);
+
+  Type0         *dummy0;
+
+  int num_reservoirs;
+  int i;
+
+  if (public_xtra)
+  {
+    NA_FreeNameArray(public_xtra->reservoir_names);
+
+    /* Free the reservoir information */
+    num_reservoirs = (public_xtra->num_reservoirs);
+    if (num_reservoirs > 0)
+    {
+      for (i = 0; i < num_reservoirs; i++)
+      {
+        dummy0 = (Type0*)(public_xtra->data[i]);
+
+        if ((dummy0->name))
+        {
+          tfree((dummy0->name));
+        }
+      }
+      tfree(public_xtra->data);
+    }
+    tfree(public_xtra);
+  }
+}
+
+
+/*--------------------------------------------------------------------------
+ * ReservoirPackageSizeOfTempData
+ *--------------------------------------------------------------------------*/
+
+int  ReservoirPackageSizeOfTempData()
+{
+  return 0;
+}
diff --git a/pfsimulator/parflow_lib/richards_jacobian_eval.c b/pfsimulator/parflow_lib/richards_jacobian_eval.c
index 9db749277..53d743764 100644
--- a/pfsimulator/parflow_lib/richards_jacobian_eval.c
+++ b/pfsimulator/parflow_lib/richards_jacobian_eval.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*
  * SGS TODO this needs some work in the overland flow current
@@ -61,7 +61,8 @@ typedef struct {
   enum JacobianType type;
   double SpinupDampP1; // NBE
   double SpinupDampP2; // NBE
-  int tfgupwind;  // @RMM
+  int tfgupwind;  // RMM
+  int using_MGSemi;  // RMM
 } PublicXtra;
 
 typedef struct {
@@ -200,7 +201,7 @@ void    RichardsJacobianEval(
                                                       * symmetric part of the Jacobian (1), or the
                                                       * full Jacobian */
 {
-  PUSH_NVTX("RichardsJacobianEval",1)
+  PUSH_NVTX("RichardsJacobianEval", 1)
 
   PFModule      *this_module = ThisPFModule;
   InstanceXtra  *instance_xtra = (InstanceXtra*)PFModuleInstanceXtra(this_module);
@@ -247,7 +248,7 @@ void    RichardsJacobianEval(
   Vector      *x_ssl = ProblemDataSSlopeX(problem_data);               //@RMM
   Vector      *y_ssl = ProblemDataSSlopeY(problem_data);               //@RMM
   Subvector   *x_ssl_sub, *y_ssl_sub;    //@RMM
-  double      *x_ssl_dat=NULL, *y_ssl_dat=NULL;     //@RMM
+  double      *x_ssl_dat = NULL, *y_ssl_dat = NULL;     //@RMM
 
   /* @RMM variable dz multiplier */
   Vector      *z_mult = ProblemDataZmult(problem_data);              //@RMM
@@ -329,6 +330,9 @@ void    RichardsJacobianEval(
           case OverlandDiffusive:
           {
             public_xtra->type = overland_flow;
+            /*If we have MGSemi set as the preconditioner key
+             * we still set the type to overland_flow
+             * but later use the simple/symmetric preconditioner (RMM) */
           }
           break;
         }
@@ -486,12 +490,12 @@ void    RichardsJacobianEval(
       ForPatchCellsPerFace(DirichletBC,
                            BeforeAllCells(DoNothing),
                            LoopVars(i, j, k, ival, bc_struct, ipatch, is),
-                           Locals(int pp_idx, ip; double value;),
+                           Locals(int pp_idx, ip; double value; ),
                            CellSetup({
-                               pp_idx = 0;
-                               ip = SubvectorEltIndex(p_sub, i, j, k);
-                               value = bc_patch_values[ival];
-                             }),
+        pp_idx = 0;
+        ip = SubvectorEltIndex(p_sub, i, j, k);
+        value = bc_patch_values[ival];
+      }),
                            FACE(LeftFace, { pp_idx = ip - 1; }),
                            FACE(RightFace, { pp_idx = ip + 1; }),
                            FACE(DownFace, { pp_idx = ip - sy_v; }),
@@ -500,7 +504,7 @@ void    RichardsJacobianEval(
                            FACE(FrontFace, { pp_idx = ip + sz_v; }),
                            CellFinalize({ pp[pp_idx] = value; }),
                            AfterAllCells(DoNothing)
-        ); /* End DirichletBC Case */
+                           ); /* End DirichletBC Case */
     }          /* End ipatch loop */
   }            /* End subgrid loop */
 
@@ -610,13 +614,13 @@ void    RichardsJacobianEval(
 
       double prod_no = rpp[ip + sy_v] * dp[ip + sy_v];
       double prod_no_der = rpdp[ip + sy_v] * dp[ip + sy_v]
-                    + rpp[ip + sy_v] * ddp[ip + sy_v];
+                           + rpp[ip + sy_v] * ddp[ip + sy_v];
 
       double prod_up = rpp[ip + sz_v] * dp[ip + sz_v];
       double prod_up_der = rpdp[ip + sz_v] * dp[ip + sz_v]
-                    + rpp[ip + sz_v] * ddp[ip + sz_v];
+                           + rpp[ip + sz_v] * ddp[ip + sz_v];
 
-      //@RMM  tfgupwind == 0 (default) should give original behavior
+      //RMM  tfgupwind == 0 (default) should give original behavior
       // tfgupwind 1 should still use sine but upwind
       // tfgupwdin 2 just upwind
 
@@ -627,35 +631,35 @@ void    RichardsJacobianEval(
 
       switch (public_xtra->tfgupwind)
       {
-        case 0:
-          {
-            // default formulation in Maxwell 2013
-            x_dir_g = Mean(gravity * sin(atan(x_ssl_dat[ioo])), gravity * sin(atan(x_ssl_dat[ioo + 1])));
-            x_dir_g_c = Mean(gravity * cos(atan(x_ssl_dat[ioo])), gravity * cos(atan(x_ssl_dat[ioo + 1])));
-            y_dir_g = Mean(gravity * sin(atan(y_ssl_dat[ioo])), gravity * sin(atan(y_ssl_dat[ioo + sy_v])));
-            y_dir_g_c = Mean(gravity * cos(atan(y_ssl_dat[ioo])), gravity * cos(atan(y_ssl_dat[ioo + sy_v])));
-            break;
-          }
-
-        case 1:
-          {
-            // direct upwinding, no averaging with sines
-            x_dir_g = gravity * sin(atan(x_ssl_dat[ioo]));
-            x_dir_g_c = gravity * cos(atan(x_ssl_dat[ioo]));
-            y_dir_g = gravity * sin(atan(y_ssl_dat[ioo]));
-            y_dir_g_c = gravity * cos(atan(y_ssl_dat[ioo]));
-            break;
-          }
-
-        case 2:
-          {
-            // direct upwinding, no averaging no sines
-            x_dir_g = x_ssl_dat[ioo];
-            x_dir_g_c = 1.0;
-            y_dir_g = y_ssl_dat[ioo];
-            y_dir_g_c = 1.0;
-            break;
-          }
+          case 0:
+            {
+              // default formulation in Maxwell 2013
+              x_dir_g = Mean(gravity * sin(atan(x_ssl_dat[ioo])), gravity * sin(atan(x_ssl_dat[ioo + 1])));
+              x_dir_g_c = Mean(gravity * cos(atan(x_ssl_dat[ioo])), gravity * cos(atan(x_ssl_dat[ioo + 1])));
+              y_dir_g = Mean(gravity * sin(atan(y_ssl_dat[ioo])), gravity * sin(atan(y_ssl_dat[ioo + sy_v])));
+              y_dir_g_c = Mean(gravity * cos(atan(y_ssl_dat[ioo])), gravity * cos(atan(y_ssl_dat[ioo + sy_v])));
+              break;
+            }
+
+          case 1:
+            {
+              // direct upwinding, no averaging with sines
+              x_dir_g = gravity * sin(atan(x_ssl_dat[ioo]));
+              x_dir_g_c = gravity * cos(atan(x_ssl_dat[ioo]));
+              y_dir_g = gravity * sin(atan(y_ssl_dat[ioo]));
+              y_dir_g_c = gravity * cos(atan(y_ssl_dat[ioo]));
+              break;
+            }
+
+          case 2:
+            {
+              // direct upwinding, no averaging no sines
+              x_dir_g = x_ssl_dat[ioo];
+              x_dir_g_c = 1.0;
+              y_dir_g = y_ssl_dat[ioo];
+              y_dir_g_c = 1.0;
+              break;
+            }
       }
 
 
@@ -665,28 +669,28 @@ void    RichardsJacobianEval(
 
       /* multiply X_coeff by FB in x */
       double x_coeff = FBx_dat[ip] * dt * ffx * (1.0 / dx) * z_mult_dat[ip]
-                * PMean(pp[ip], pp[ip + 1], permxp[ip], permxp[ip + 1])
-                / viscosity;
+                       * PMean(pp[ip], pp[ip + 1], permxp[ip], permxp[ip + 1])
+                       / viscosity;
 
 
       double sym_west_temp = (-x_coeff
-                       * RPMean(updir, 0.0, prod, prod_rt)) * x_dir_g_c; //@RMM TFG contributions, sym
+                              * RPMean(updir, 0.0, prod, prod_rt)) * x_dir_g_c; //RMM TFG contributions, sym
 
 
       double west_temp = (-x_coeff * diff
-                   * RPMean(updir, 0.0, prod_der, 0.0)) * x_dir_g_c
-                  + sym_west_temp;
+                          * RPMean(updir, 0.0, prod_der, 0.0)) * x_dir_g_c
+                         + sym_west_temp;
 
-      west_temp += (x_coeff * dx * RPMean(updir, 0.0, prod_der, 0.0)) * x_dir_g; //@RMM TFG contributions, non sym
+      west_temp += (x_coeff * dx * RPMean(updir, 0.0, prod_der, 0.0)) * x_dir_g; //RMM TFG contributions, non sym
 
       double sym_east_temp = (-x_coeff
-                       * RPMean(updir, 0.0, prod, prod_rt)) * x_dir_g_c; //@RMM added sym TFG contributions
+                              * RPMean(updir, 0.0, prod, prod_rt)) * x_dir_g_c; //RMM added sym TFG contributions
 
       double east_temp = (x_coeff * diff
-                   * RPMean(updir, 0.0, 0.0, prod_rt_der)) * x_dir_g_c
-                  + sym_east_temp;
+                          * RPMean(updir, 0.0, 0.0, prod_rt_der)) * x_dir_g_c
+                         + sym_east_temp;
 
-      east_temp += -(x_coeff * dx * RPMean(updir, 0.0, 0.0, prod_rt_der)) * x_dir_g; //@RMM  TFG contributions non sym
+      east_temp += -(x_coeff * dx * RPMean(updir, 0.0, 0.0, prod_rt_der)) * x_dir_g; //RMM  TFG contributions non sym
 
       /* diff >= 0 implies flow goes south to north */
       diff = pp[ip] - pp[ip + sy_v];
@@ -695,28 +699,28 @@ void    RichardsJacobianEval(
 
       /* multiply y_coeff by FB in y */
       double y_coeff = FBy_dat[ip] * dt * ffy * (1.0 / dy) * z_mult_dat[ip]
-                * PMean(pp[ip], pp[ip + sy_v], permyp[ip], permyp[ip + sy_v])
-                / viscosity;
+                       * PMean(pp[ip], pp[ip + sy_v], permyp[ip], permyp[ip + sy_v])
+                       / viscosity;
 
       double sym_south_temp = -y_coeff
-                       * RPMean(updir, 0.0, prod, prod_no) * y_dir_g_c; //@RMM TFG contributions, SYMM
+                              * RPMean(updir, 0.0, prod, prod_no) * y_dir_g_c; //RMM TFG contributions, SYMM
 
       double south_temp = -y_coeff * diff
-                   * RPMean(updir, 0.0, prod_der, 0.0) * y_dir_g_c
-                   + sym_south_temp;
+                          * RPMean(updir, 0.0, prod_der, 0.0) * y_dir_g_c
+                          + sym_south_temp;
 
-      south_temp += (y_coeff * dy * RPMean(updir, 0.0, prod_der, 0.0)) * y_dir_g; //@RMM TFG contributions, non sym
+      south_temp += (y_coeff * dy * RPMean(updir, 0.0, prod_der, 0.0)) * y_dir_g; //RMM TFG contributions, non sym
 
 
       double sym_north_temp = y_coeff
-                       * -RPMean(updir, 0.0, prod, prod_no) * y_dir_g_c; //@RMM  TFG contributions non SYMM
+                              * -RPMean(updir, 0.0, prod, prod_no) * y_dir_g_c; //RMM  TFG contributions non SYMM
 
       double north_temp = y_coeff * diff
-                   * RPMean(updir, 0.0, 0.0,
-                            prod_no_der) * y_dir_g_c
-                   + sym_north_temp;
+                          * RPMean(updir, 0.0, 0.0,
+                                   prod_no_der) * y_dir_g_c
+                          + sym_north_temp;
 
-      north_temp += -(y_coeff * dy * RPMean(updir, 0.0, 0.0, prod_no_der)) * y_dir_g; //@RMM  TFG contributions non sym
+      north_temp += -(y_coeff * dy * RPMean(updir, 0.0, 0.0, prod_no_der)) * y_dir_g; //RMM  TFG contributions non sym
 
       double sep = (dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v]));
       /* diff >= 0 implies flow goes lower to upper */
@@ -731,31 +735,31 @@ void    RichardsJacobianEval(
 
       /* multiply z_coeff by FB in z */
       double z_coeff = FBz_dat[ip] * dt * ffz
-                * PMeanDZ(permzp[ip], permzp[ip + sz_v], z_mult_dat[ip], z_mult_dat[ip + sz_v])
-                / viscosity;
+                       * PMeanDZ(permzp[ip], permzp[ip + sz_v], z_mult_dat[ip], z_mult_dat[ip + sz_v])
+                       / viscosity;
 
       double sym_lower_temp = -z_coeff * (1.0 / (dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])))
-                       * RPMean(lower_cond, upper_cond, prod,
-                                prod_up);
+                              * RPMean(lower_cond, upper_cond, prod,
+                                       prod_up);
 
       double lower_temp = -z_coeff
-                   * (diff * RPMean(lower_cond, upper_cond, prod_der, 0.0)
-                      + (-gravity * 0.5 * dz * (Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])) * ddp[ip]
-                         * RPMean(lower_cond, upper_cond, prod,
-                                  prod_up)))
-                   + sym_lower_temp;
+                          * (diff * RPMean(lower_cond, upper_cond, prod_der, 0.0)
+                             + (-gravity * 0.5 * dz * (Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])) * ddp[ip]
+                                * RPMean(lower_cond, upper_cond, prod,
+                                         prod_up)))
+                          + sym_lower_temp;
 
       double sym_upper_temp = z_coeff * (1.0 / (dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])))
-                       * -RPMean(lower_cond, upper_cond, prod,
-                                 prod_up);
+                              * -RPMean(lower_cond, upper_cond, prod,
+                                        prod_up);
 
       double upper_temp = z_coeff
-                   * (diff * RPMean(lower_cond, upper_cond, 0.0,
-                                    prod_up_der)
-                      + (-gravity * 0.5 * dz * (Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])) * ddp[ip + sz_v]
-                         * RPMean(lower_cond, upper_cond, prod,
-                                  prod_up)))
-                   + sym_upper_temp;
+                          * (diff * RPMean(lower_cond, upper_cond, 0.0,
+                                           prod_up_der)
+                             + (-gravity * 0.5 * dz * (Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])) * ddp[ip + sz_v]
+                                * RPMean(lower_cond, upper_cond, prod,
+                                         prod_up)))
+                          + sym_upper_temp;
 
       PlusEquals(cp[im], -(west_temp + south_temp + lower_temp));
       PlusEquals(cp[im + 1], -east_temp);
@@ -820,9 +824,9 @@ void    RichardsJacobianEval(
 
       sy_v = nx_v;
       sz_v = ny_v * nx_v;
-      /* @RMM added to provide access to zmult */
+      /* RMM added to provide access to zmult */
       z_mult_sub = VectorSubvector(z_mult, is);
-      /* @RMM added to provide variable dz */
+      /* RMM added to provide variable dz */
       z_mult_dat = SubvectorData(z_mult_sub);
 
       cp = SubmatrixStencilData(J_sub, 0);
@@ -850,114 +854,113 @@ void    RichardsJacobianEval(
                              Locals(int ip, im;
                                     double coeff, diff, prod, prod_der;
                                     double lower_cond, upper_cond;
-                                    double prod_lo, prod_up;),
+                                    double prod_lo, prod_up; ),
                              CellSetup({
-                                 ip = SubvectorEltIndex(p_sub, i, j, k);
-                                 im = SubmatrixEltIndex(J_sub, i, j, k);
-
-                                 /* SGS added this as prod was not being set to anything. Check with carol. */
-                                 prod = rpp[ip] * dp[ip];
-                                 prod_der = 0.0;
-                                 prod_lo = 0.0;
-                                 prod_up = 0.0;
-                                 coeff = 0.0;
-                                 diff = 0.0;
-                                 lower_cond = 0.0;
-                                 upper_cond = 0.0;
-                               }),
+          ip = SubvectorEltIndex(p_sub, i, j, k);
+          im = SubmatrixEltIndex(J_sub, i, j, k);
+
+          /* SGS added this as prod was not being set to anything. Check with carol. */
+          prod = rpp[ip] * dp[ip];
+          prod_der = 0.0;
+          prod_lo = 0.0;
+          prod_up = 0.0;
+          coeff = 0.0;
+          diff = 0.0;
+          lower_cond = 0.0;
+          upper_cond = 0.0;
+        }),
                              FACE(LeftFace,
-                             {
-                               diff = pp[ip - 1] - pp[ip];
-                               prod_der = rpdp[ip - 1] * dp[ip - 1] + rpp[ip - 1] * ddp[ip - 1];
-                               coeff = dt * z_mult_dat[ip] * ffx * (1.0 / dx)
-                                       * PMean(pp[ip - 1], pp[ip], permxp[ip - 1], permxp[ip])
-                                       / viscosity;
-                               wp[im] = -coeff * diff
-                                        * RPMean(pp[ip - 1], pp[ip], prod_der, 0.0);
-                             }),
+        {
+          diff = pp[ip - 1] - pp[ip];
+          prod_der = rpdp[ip - 1] * dp[ip - 1] + rpp[ip - 1] * ddp[ip - 1];
+          coeff = dt * z_mult_dat[ip] * ffx * (1.0 / dx)
+                  * PMean(pp[ip - 1], pp[ip], permxp[ip - 1], permxp[ip])
+                  / viscosity;
+          wp[im] = -coeff * diff
+                   * RPMean(pp[ip - 1], pp[ip], prod_der, 0.0);
+        }),
                              FACE(RightFace,
-                             {
-                               diff = pp[ip] - pp[ip + 1];
-                               prod_der = rpdp[ip + 1] * dp[ip + 1] + rpp[ip + 1] * ddp[ip + 1];
-                               coeff = dt * z_mult_dat[ip] * ffx * (1.0 / dx)
-                                       * PMean(pp[ip], pp[ip + 1], permxp[ip], permxp[ip + 1])
-                                       / viscosity;
-                               ep[im] = coeff * diff
-                                        * RPMean(pp[ip], pp[ip + 1], 0.0, prod_der);
-                             }),
+        {
+          diff = pp[ip] - pp[ip + 1];
+          prod_der = rpdp[ip + 1] * dp[ip + 1] + rpp[ip + 1] * ddp[ip + 1];
+          coeff = dt * z_mult_dat[ip] * ffx * (1.0 / dx)
+                  * PMean(pp[ip], pp[ip + 1], permxp[ip], permxp[ip + 1])
+                  / viscosity;
+          ep[im] = coeff * diff
+                   * RPMean(pp[ip], pp[ip + 1], 0.0, prod_der);
+        }),
                              FACE(DownFace,
-                             {
-                               diff = pp[ip - sy_v] - pp[ip];
-                               prod_der = rpdp[ip - sy_v] * dp[ip - sy_v]
-                                          + rpp[ip - sy_v] * ddp[ip - sy_v];
-                               coeff = dt * z_mult_dat[ip] * ffy * (1.0 / dy)
-                                       * PMean(pp[ip - sy_v], pp[ip],
-                                               permyp[ip - sy_v], permyp[ip])
-                                       / viscosity;
-                               sop[im] = -coeff * diff
-                                         * RPMean(pp[ip - sy_v], pp[ip], prod_der, 0.0);
-                             }),
+        {
+          diff = pp[ip - sy_v] - pp[ip];
+          prod_der = rpdp[ip - sy_v] * dp[ip - sy_v]
+                     + rpp[ip - sy_v] * ddp[ip - sy_v];
+          coeff = dt * z_mult_dat[ip] * ffy * (1.0 / dy)
+                  * PMean(pp[ip - sy_v], pp[ip],
+                          permyp[ip - sy_v], permyp[ip])
+                  / viscosity;
+          sop[im] = -coeff * diff
+                    * RPMean(pp[ip - sy_v], pp[ip], prod_der, 0.0);
+        }),
                              FACE(UpFace,
-                             {
-                               diff = pp[ip] - pp[ip + sy_v];
-                               prod_der = rpdp[ip + sy_v] * dp[ip + sy_v]
-                                          + rpp[ip + sy_v] * ddp[ip + sy_v];
-                               coeff = dt * z_mult_dat[ip] * ffy * (1.0 / dy)
-                                       * PMean(pp[ip], pp[ip + sy_v],
-                                               permyp[ip], permyp[ip + sy_v])
-                                       / viscosity;
-                               np[im] = -coeff * diff
-                                        * RPMean(pp[ip], pp[ip + sy_v], 0.0, prod_der);
-                             }),
+        {
+          diff = pp[ip] - pp[ip + sy_v];
+          prod_der = rpdp[ip + sy_v] * dp[ip + sy_v]
+                     + rpp[ip + sy_v] * ddp[ip + sy_v];
+          coeff = dt * z_mult_dat[ip] * ffy * (1.0 / dy)
+                  * PMean(pp[ip], pp[ip + sy_v],
+                          permyp[ip], permyp[ip + sy_v])
+                  / viscosity;
+          np[im] = -coeff * diff
+                   * RPMean(pp[ip], pp[ip + sy_v], 0.0, prod_der);
+        }),
                              FACE(BackFace,
-                             {
-                               lower_cond = (pp[ip - sz_v]) - 0.5 * dz
-                                            * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_v])
-                                            * dp[ip - sz_v] * gravity;
-                               upper_cond = (pp[ip]) + 0.5 * dz * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_v])
-                                            * dp[ip] * gravity;
-                               diff = lower_cond - upper_cond;
-                               prod_der = rpdp[ip - sz_v] * dp[ip - sz_v]
-                                          + rpp[ip - sz_v] * ddp[ip - sz_v];
-                               prod_lo = rpp[ip - sz_v] * dp[ip - sz_v];
-                               coeff = dt * ffz * (1.0 / (dz * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_v])))
-                                       * PMeanDZ(permzp[ip - sz_v], permzp[ip],
-                                                 z_mult_dat[ip - sz_v], z_mult_dat[ip])
-                                       / viscosity;
-                               lp[im] = -coeff *
-                                        (diff * RPMean(lower_cond, upper_cond,
-                                                       prod_der, 0.0)
-                                         - gravity * 0.5 * dz
-                                         * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_v]) * ddp[ip]
-                                         * RPMean(lower_cond, upper_cond, prod_lo, prod));
-                             }),
+        {
+          lower_cond = (pp[ip - sz_v]) - 0.5 * dz
+                       * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_v])
+                       * dp[ip - sz_v] * gravity;
+          upper_cond = (pp[ip]) + 0.5 * dz * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_v])
+                       * dp[ip] * gravity;
+          diff = lower_cond - upper_cond;
+          prod_der = rpdp[ip - sz_v] * dp[ip - sz_v]
+                     + rpp[ip - sz_v] * ddp[ip - sz_v];
+          prod_lo = rpp[ip - sz_v] * dp[ip - sz_v];
+          coeff = dt * ffz * (1.0 / (dz * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_v])))
+                  * PMeanDZ(permzp[ip - sz_v], permzp[ip],
+                            z_mult_dat[ip - sz_v], z_mult_dat[ip])
+                  / viscosity;
+          lp[im] = -coeff *
+                   (diff * RPMean(lower_cond, upper_cond,
+                                  prod_der, 0.0)
+                    - gravity * 0.5 * dz
+                    * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_v]) * ddp[ip]
+                    * RPMean(lower_cond, upper_cond, prod_lo, prod));
+        }),
                              FACE(FrontFace,
-                             {
-                               lower_cond = (pp[ip]) - 0.5 * dz
-                                            * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])
-                                            * dp[ip] * gravity;
-                               upper_cond = (pp[ip + sz_v]) + 0.5 * dz
-                                            * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])
-                                            * dp[ip + sz_v] * gravity;
-                               diff = lower_cond - upper_cond;
-                               prod_der = rpdp[ip + sz_v] * dp[ip + sz_v]
-                                          + rpp[ip + sz_v] * ddp[ip + sz_v];
-                               prod_up = rpp[ip + sz_v] * dp[ip + sz_v];
-                               coeff = dt * ffz * (1.0 / (dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])))
-                                       * PMeanDZ(permzp[ip], permzp[ip + sz_v],
-                                                 z_mult_dat[ip], z_mult_dat[ip + sz_v])
-                                       / viscosity;
-                               up[im] = -coeff *
-                                        (diff * RPMean(lower_cond, upper_cond,
-                                                       0.0, prod_der)
-                                         - gravity * 0.5 * dz
-                                         * (Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])) * ddp[ip]
-                                         * RPMean(lower_cond, upper_cond, prod, prod_up));
-
-                             }),
+        {
+          lower_cond = (pp[ip]) - 0.5 * dz
+                       * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])
+                       * dp[ip] * gravity;
+          upper_cond = (pp[ip + sz_v]) + 0.5 * dz
+                       * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])
+                       * dp[ip + sz_v] * gravity;
+          diff = lower_cond - upper_cond;
+          prod_der = rpdp[ip + sz_v] * dp[ip + sz_v]
+                     + rpp[ip + sz_v] * ddp[ip + sz_v];
+          prod_up = rpp[ip + sz_v] * dp[ip + sz_v];
+          coeff = dt * ffz * (1.0 / (dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])))
+                  * PMeanDZ(permzp[ip], permzp[ip + sz_v],
+                            z_mult_dat[ip], z_mult_dat[ip + sz_v])
+                  / viscosity;
+          up[im] = -coeff *
+                   (diff * RPMean(lower_cond, upper_cond,
+                                  0.0, prod_der)
+                    - gravity * 0.5 * dz
+                    * (Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])) * ddp[ip]
+                    * RPMean(lower_cond, upper_cond, prod, prod_up));
+        }),
                              CellFinalize(DoNothing),
                              AfterAllCells(DoNothing)
-          ); /* End Patch Loop */
+                             ); /* End Patch Loop */
       }           /* End ipatch loop */
     }             /* End subgrid loop */
   }                  /* End if symm_part */
@@ -991,9 +994,9 @@ void    RichardsJacobianEval(
     dy = SubgridDY(subgrid);
     dz = SubgridDZ(subgrid);
 
-    /* @RMM added to provide access to zmult */
+    /* RMM added to provide access to zmult */
     z_mult_sub = VectorSubvector(z_mult, is);
-    /* @RMM added to provide variable dz */
+    /* RMM added to provide variable dz */
     z_mult_dat = SubvectorData(z_mult_sub);
 
     double vol = dx * dy * dz;
@@ -1051,342 +1054,343 @@ void    RichardsJacobianEval(
 
       ForPatchCellsPerFace(DirichletBC,
                            BeforeAllCells(
-                           {
-                             /* @MCB 04/14/2020:
-                                Previously two module invokes were made every iteration
-                                of the loop.  However, these calls were only retrieving
-                                a scalar and (potentially) multipling it against the BC
-                                value.
-                                The PhaseDensityConstants function was added to
-                                retrieve those values once and set den_d and dend_d appropriately.
-                             */
-
-                             ThisPFModule = density_module;
-                             PhaseDensityConstants(0, CALCFCN, &phase_type,
-                                                   &fcn_phase_const,
-                                                   &phase_ref,
-                                                   &phase_comp);
-                             PhaseDensityConstants(0, CALCDER, &phase_type,
-                                                   &der_phase_const,
-                                                   &phase_ref,
-                                                   &phase_comp);
-                           }),
+      {
+        /* MCB 04/14/2020:
+         * Previously two module invokes were made every iteration
+         * of the loop.  However, these calls were only retrieving
+         * a scalar and (potentially) multipling it against the BC
+         * value.
+         * The PhaseDensityConstants function was added to
+         * retrieve those values once and set den_d and dend_d appropriately.
+         */
+
+        ThisPFModule = density_module;
+        PhaseDensityConstants(0, CALCFCN, &phase_type,
+                              &fcn_phase_const,
+                              &phase_ref,
+                              &phase_comp);
+        PhaseDensityConstants(0, CALCDER, &phase_type,
+                              &der_phase_const,
+                              &phase_ref,
+                              &phase_comp);
+      }),
                            LoopVars(i, j, k, ival, bc_struct, ipatch, is),
                            Locals(int ip, im;
                                   double *op;
                                   double den_d, value, o_temp;
                                   double prod, prod_der, prod_val;
-                                  double diff, coeff, lower_cond, upper_cond;),
+                                  double diff, coeff, lower_cond, upper_cond; ),
                            CellSetup(
-                           {
-                             ip = SubvectorEltIndex(p_sub, i, j, k);
-                             im = SubmatrixEltIndex(J_sub, i, j, k);
-                             value = bc_patch_values[ival];
-
-                             op = NULL;
-                             coeff = 0.0;
-                             diff = 0.0;
-                             o_temp = 0.0;
-                             prod_val = 0.0;
-                             lower_cond = 0.0;
-                             upper_cond = 0.0;
-
-							 /* Note: If dend_d is needed, redeclare in Locals block above */
-                             if (phase_type == 0) {
-                               den_d = fcn_phase_const;
-                               //dend_d = der_phase_const;
-                             } else {
-                               den_d = phase_ref * exp(value * phase_comp);
-                               //dend_d = phase_comp * phase_ref * exp(value * phase_comp);
-                             }
-
-                             prod = rpp[ip] * dp[ip];
-                             prod_der = rpdp[ip] * dp[ip] + rpp[ip] * ddp[ip];
-                           }),
+      {
+        ip = SubvectorEltIndex(p_sub, i, j, k);
+        im = SubmatrixEltIndex(J_sub, i, j, k);
+        value = bc_patch_values[ival];
+
+        op = NULL;
+        coeff = 0.0;
+        diff = 0.0;
+        o_temp = 0.0;
+        prod_val = 0.0;
+        lower_cond = 0.0;
+        upper_cond = 0.0;
+
+        /* Note: If dend_d is needed, redeclare in Locals block above */
+        if (phase_type == 0)
+        {
+          den_d = fcn_phase_const;
+          //dend_d = der_phase_const;
+        }
+        else
+        {
+          den_d = phase_ref * exp(value * phase_comp);
+          //dend_d = phase_comp * phase_ref * exp(value * phase_comp);
+        }
+
+        prod = rpp[ip] * dp[ip];
+        prod_der = rpdp[ip] * dp[ip] + rpp[ip] * ddp[ip];
+      }),
                            FACE(LeftFace,
-                           {
-                             op = wp;
-                             coeff = dt * ffx * z_mult_dat[ip] * (2.0 / dx) * permxp[ip] / viscosity;
-                             prod_val = rpp[ip - 1] * den_d;
-                             diff = value - pp[ip];
-                             o_temp = coeff
-                                      * (diff * RPMean(value, pp[ip], 0.0, prod_der)
-                                         - RPMean(value, pp[ip], prod_val, prod));
-                           }),
+      {
+        op = wp;
+        coeff = dt * ffx * z_mult_dat[ip] * (2.0 / dx) * permxp[ip] / viscosity;
+        prod_val = rpp[ip - 1] * den_d;
+        diff = value - pp[ip];
+        o_temp = coeff
+                 * (diff * RPMean(value, pp[ip], 0.0, prod_der)
+                    - RPMean(value, pp[ip], prod_val, prod));
+      }),
                            FACE(RightFace,
-                           {
-                             op = ep;
-                             coeff = dt * ffx * z_mult_dat[ip] * (2.0 / dx) * permxp[ip] / viscosity;
-                             prod_val = rpp[ip + 1] * den_d;
-                             diff = pp[ip] - value;
-                             o_temp = -coeff
-                                      * (diff * RPMean(pp[ip], value, prod_der, 0.0)
-                                         + RPMean(pp[ip], value, prod, prod_val));
-                           }),
+      {
+        op = ep;
+        coeff = dt * ffx * z_mult_dat[ip] * (2.0 / dx) * permxp[ip] / viscosity;
+        prod_val = rpp[ip + 1] * den_d;
+        diff = pp[ip] - value;
+        o_temp = -coeff
+                 * (diff * RPMean(pp[ip], value, prod_der, 0.0)
+                    + RPMean(pp[ip], value, prod, prod_val));
+      }),
                            FACE(DownFace,
-                           {
-                             op = sop;
-                             coeff = dt * ffy * z_mult_dat[ip] * (2.0 / dy) * permyp[ip] / viscosity;
-                             prod_val = rpp[ip - sy_v] * den_d;
-                             diff = value - pp[ip];
-                             o_temp = coeff
-                                      * (diff * RPMean(value, pp[ip], 0.0, prod_der)
-                                         - RPMean(value, pp[ip], prod_val, prod));
-                           }),
+      {
+        op = sop;
+        coeff = dt * ffy * z_mult_dat[ip] * (2.0 / dy) * permyp[ip] / viscosity;
+        prod_val = rpp[ip - sy_v] * den_d;
+        diff = value - pp[ip];
+        o_temp = coeff
+                 * (diff * RPMean(value, pp[ip], 0.0, prod_der)
+                    - RPMean(value, pp[ip], prod_val, prod));
+      }),
                            FACE(UpFace,
-                           {
-                             op = np;
-                             coeff = dt * ffy * z_mult_dat[ip] * (2.0 / dy) * permyp[ip] / viscosity;
-                             prod_val = rpp[ip + sy_v] * den_d;
-                             diff = pp[ip] - value;
-                             o_temp = -coeff
-                                      * (diff * RPMean(pp[ip], value, prod_der, 0.0)
-                                         + RPMean(pp[ip], value, prod, prod_val));
-                           }),
+      {
+        op = np;
+        coeff = dt * ffy * z_mult_dat[ip] * (2.0 / dy) * permyp[ip] / viscosity;
+        prod_val = rpp[ip + sy_v] * den_d;
+        diff = pp[ip] - value;
+        o_temp = -coeff
+                 * (diff * RPMean(pp[ip], value, prod_der, 0.0)
+                    + RPMean(pp[ip], value, prod, prod_val));
+      }),
                            FACE(BackFace,
-                           {
-                             coeff = dt * ffz * (2.0 / (dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])))
-                                     * permzp[ip] / viscosity;
-                             op = lp;
-                             prod_val = rpp[ip - sz_v] * den_d;
+      {
+        coeff = dt * ffz * (2.0 / (dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])))
+                * permzp[ip] / viscosity;
+        op = lp;
+        prod_val = rpp[ip - sz_v] * den_d;
 
-                             lower_cond = (value) - 0.5 * dz * z_mult_dat[ip] * den_d * gravity;
-                             upper_cond = (pp[ip]) + 0.5 * dz * z_mult_dat[ip] * dp[ip] * gravity;
-                             diff = lower_cond - upper_cond;
+        lower_cond = (value) - 0.5 * dz * z_mult_dat[ip] * den_d * gravity;
+        upper_cond = (pp[ip]) + 0.5 * dz * z_mult_dat[ip] * dp[ip] * gravity;
+        diff = lower_cond - upper_cond;
 
 //                    o_temp = coeff
 //                             * (diff * RPMean(lower_cond, upper_cond, 0.0, prod_der)
 //                                + ((-1.0 - gravity * 0.5 * dz * Mean(z_mult_dat[ip], z_mult_dat[ip - sz_v]) * ddp[ip])
 //                                   * RPMean(lower_cond, upper_cond, prod_val, prod)));
 
-                             o_temp = coeff
-                                      * (diff * RPMean(lower_cond, upper_cond, 0.0, prod_der)
-                                         + ((-1.0 - gravity * 0.5 * dz * z_mult_dat[ip] * ddp[ip])
-                                            * RPMean(lower_cond, upper_cond, prod_val, prod)));
+        o_temp = coeff
+                 * (diff * RPMean(lower_cond, upper_cond, 0.0, prod_der)
+                    + ((-1.0 - gravity * 0.5 * dz * z_mult_dat[ip] * ddp[ip])
+                       * RPMean(lower_cond, upper_cond, prod_val, prod)));
 
 //printf("jacobian lower BC: o_temp=%f prod_der=%f op=%f \n",o_temp, prod_der, op);
-                           }),
+      }),
                            FACE(FrontFace,
-                           {
-                             coeff = dt * ffz * (2.0 / (dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])))
-                                     * permzp[ip] / viscosity;
+      {
+        coeff = dt * ffz * (2.0 / (dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])))
+                * permzp[ip] / viscosity;
 
-                             op = up;
-                             prod_val = rpp[ip + sz_v] * den_d;
+        op = up;
+        prod_val = rpp[ip + sz_v] * den_d;
 
-                             lower_cond = (pp[ip]) - 0.5 * dz * z_mult_dat[ip] * dp[ip] * gravity;
-                             upper_cond = (value) + 0.5 * dz * z_mult_dat[ip] * den_d * gravity;
-                             diff = lower_cond - upper_cond;
+        lower_cond = (pp[ip]) - 0.5 * dz * z_mult_dat[ip] * dp[ip] * gravity;
+        upper_cond = (value) + 0.5 * dz * z_mult_dat[ip] * den_d * gravity;
+        diff = lower_cond - upper_cond;
 
-                             o_temp = -coeff * (diff * RPMean(lower_cond, upper_cond, prod_der, 0.0)
-                                                + ((1.0 - gravity * 0.5 * dz * z_mult_dat[ip] * ddp[ip])
-                                                   * RPMean(lower_cond, upper_cond, prod, prod_val)));
-                           }),
+        o_temp = -coeff * (diff * RPMean(lower_cond, upper_cond, prod_der, 0.0)
+                           + ((1.0 - gravity * 0.5 * dz * z_mult_dat[ip] * ddp[ip])
+                              * RPMean(lower_cond, upper_cond, prod, prod_val)));
+      }),
                            CellFinalize(
-                           {
-                             cp[im] += op[im];
-                             cp[im] -= o_temp;
-                             op[im] = 0.0;
-                           }),
+      {
+        cp[im] += op[im];
+        cp[im] -= o_temp;
+        op[im] = 0.0;
+      }),
                            AfterAllCells(DoNothing)
-        ); /* End DirichletBC */
+                           ); /* End DirichletBC */
 
       ForPatchCellsPerFace(FluxBC,
                            BeforeAllCells(DoNothing),
                            LoopVars(i, j, k, ival, bc_struct, ipatch, is),
-                           Locals(int im; double *op;),
+                           Locals(int im; double *op; ),
                            CellSetup({ im = SubmatrixEltIndex(J_sub, i, j, k); }),
-                           FACE(LeftFace,  { op = wp; }),
+                           FACE(LeftFace, { op = wp; }),
                            FACE(RightFace, { op = ep; }),
-                           FACE(DownFace,  { op = sop; }),
-                           FACE(UpFace,    { op = np; }),
-                           FACE(BackFace,  { op = lp; }),
+                           FACE(DownFace, { op = sop; }),
+                           FACE(UpFace, { op = np; }),
+                           FACE(BackFace, { op = lp; }),
                            FACE(FrontFace, { op = up; }),
                            CellFinalize({
-                               cp[im] += op[im];
-                               op[im] = 0.0;
-                             }),
+        cp[im] += op[im];
+        op[im] = 0.0;
+      }),
                            AfterAllCells(DoNothing)
-        ); /* End FluxBC */
+                           ); /* End FluxBC */
 
       ForPatchCellsPerFace(OverlandBC,
                            BeforeAllCells(DoNothing),
                            LoopVars(i, j, k, ival, bc_struct, ipatch, is),
-                           Locals(int im, ip; double *op;),
+                           Locals(int im, ip; double *op; ),
                            CellSetup({ im = SubmatrixEltIndex(J_sub, i, j, k); }),
-                           FACE(LeftFace,  { op = wp; }),
+                           FACE(LeftFace, { op = wp; }),
                            FACE(RightFace, { op = ep; }),
-                           FACE(DownFace,  { op = sop; }),
-                           FACE(UpFace,    { op = np; }),
-                           FACE(BackFace,  { op = lp; }),
+                           FACE(DownFace, { op = sop; }),
+                           FACE(UpFace, { op = np; }),
+                           FACE(BackFace, { op = lp; }),
                            FACE(FrontFace,
-                           {
-                             op = up;
-                             if (!ovlnd_flag[0])
-                             {
-                               ip = SubvectorEltIndex(p_sub, i, j, k);
-                               if ((pp[ip]) > 0.0)
-                               {
-                                 ovlnd_flag[0] = 1;
-                               }
-                             }
-
-                             /* @MCB: This used to be after the loop.
-                                Either we would enter another BC loop, but only work on the front face
-                                Or we'd make a module call
-                                Second BC loops have been fused here, on the front face calculation.
-                                Module call has been moved to AfterAllCells
-                                This could be cleaner if the legacy flags were dealt with.
-                             */
-                             switch(public_xtra->type)
-                             {
-                               case no_nonlinear_jacobian:
-                               case not_set:
-                                 assert(1);
-
-                               case simple:
-                               {
-                                 double vol = dx * dy * dz;
-                                 int ip = SubvectorEltIndex(p_sub, i, j, k);
-                                 if ((pp[ip]) > 0.0)
-                                 {
-                                   cp[im] += (vol * z_mult_dat[ip]) / (dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])) * (dt + 1);
-                                 }
-                               }
-                               break;
-
-                               case overland_flow:
-                               {
-                                 if (overlandspinup == 1)
-                                 {
-                                   double vol = dx * dy * dz;
-                                   int ip = SubvectorEltIndex(p_sub, i, j, k);
-
-                                   if ((pp[ip]) >= 0.0)
-                                   {
-                                     cp[im] += (vol / dz) * dt * (1.0 + 0.0);                     //LEC
-//                      printf("Jac SU: CP=%f im=%d  \n", cp[im], im);
-                                   }
-                                   else
-                                   {
-                                     cp[im] += 0.0;
-                                   }
-                                 }
-                               }
-                               break;
-                             }
-                           }),
+      {
+        op = up;
+        if (!ovlnd_flag[0])
+        {
+          ip = SubvectorEltIndex(p_sub, i, j, k);
+          if ((pp[ip]) > 0.0)
+          {
+            ovlnd_flag[0] = 1;
+          }
+        }
+
+        /* MCB: This used to be after the loop.
+         * Either we would enter another BC loop, but only work on the front face
+         * Or we'd make a module call
+         * Second BC loops have been fused here, on the front face calculation.
+         * Module call has been moved to AfterAllCells
+         * This could be cleaner if the legacy flags were dealt with.
+         */
+        switch (public_xtra->type)
+        {
+            case no_nonlinear_jacobian:
+            case not_set:
+              assert(1);
+
+            case simple:
+              {
+                double vol = dx * dy * dz;
+                int ip = SubvectorEltIndex(p_sub, i, j, k);
+                if ((pp[ip]) > 0.0)
+                {
+                  cp[im] += (vol * z_mult_dat[ip]) / (dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])) * (dt + 1);
+                }
+              }
+              break;
+
+            case overland_flow:
+              {
+                if (overlandspinup == 1)
+                {
+                  double vol = dx * dy * dz;
+                  int ip = SubvectorEltIndex(p_sub, i, j, k);
+
+                  if ((pp[ip]) >= 0.0)
+                  {
+                    cp[im] += (vol / dz) * dt * (1.0 + 0.0);                                      //LEC
+                  }
+                  else
+                  {
+                    cp[im] += 0.0;
+                  }
+                }
+              }
+              break;
+        }
+      }),
                            CellFinalize({
-                               cp[im] += op[im];
-                               op[im] = 0.0;
-                             }),
+        cp[im] += op[im];
+        op[im] = 0.0;
+      }),
                            AfterAllCells(
-                           {
-                             switch(public_xtra->type)
-                             {
-                               case overland_flow:
-                                 if (overlandspinup != 1)
-                                 {
-                                   /* Get overland flow contributions for using kinematic or diffusive - LEC */
-                                   if (diffusive == 0)
-                                   {
-                                     PFModuleInvokeType(OverlandFlowEvalInvoke, overlandflow_module,
-                                                        (grid, is, bc_struct, ipatch, problem_data, pressure, old_pressure,
-                                                         ke_der, kw_der, kn_der, ks_der, NULL, NULL, CALCDER));
-                                   }
-                                   else
-                                   {
-                                     /* Test running Diffuisve calc FCN */
-                                     //double *dummy1, *dummy2, *dummy3, *dummy4;
-                                     //PFModuleInvokeType(OverlandFlowEvalDiffInvoke, overlandflow_module_diff, (grid, is, bc_struct, ipatch, problem_data, pressure,
-                                     //                                             ke_der, kw_der, kn_der, ks_der,
-                                     //       dummy1, dummy2, dummy3, dummy4,
-                                     //                                                    NULL, NULL, CALCFCN));
-
-                                     PFModuleInvokeType(OverlandFlowEvalDiffInvoke, overlandflow_module_diff,
-                                                        (grid, is, bc_struct, ipatch, problem_data, pressure, old_pressure,
-                                                         ke_der, kw_der, kn_der, ks_der,
-                                                         kens_der, kwns_der, knns_der, ksns_der, NULL, NULL, CALCDER));
-                                   }
-                                 }
-								 break;
-
-							 default:
-                                 break;
-                             }
-                           })
-        ); /* End OverlandBC */
+      {
+        switch (public_xtra->type)
+        {
+            case overland_flow:
+              if (overlandspinup != 1)
+              {
+                /* Get overland flow contributions for using kinematic or diffusive - LEC */
+                if (diffusive == 0)
+                {
+                  PFModuleInvokeType(OverlandFlowEvalInvoke, overlandflow_module,
+                                     (grid, is, bc_struct, ipatch, problem_data, pressure, old_pressure,
+                                      ke_der, kw_der, kn_der, ks_der, NULL, NULL, CALCDER));
+                }
+                else
+                {
+                  /* Test running Diffuisve calc FCN */
+                  //double *dummy1, *dummy2, *dummy3, *dummy4;
+                  //PFModuleInvokeType(OverlandFlowEvalDiffInvoke, overlandflow_module_diff, (grid, is, bc_struct, ipatch, problem_data, pressure,
+                  //                                             ke_der, kw_der, kn_der, ks_der,
+                  //       dummy1, dummy2, dummy3, dummy4,
+                  //                                                    NULL, NULL, CALCFCN));
+
+                  PFModuleInvokeType(OverlandFlowEvalDiffInvoke, overlandflow_module_diff,
+                                     (grid, is, bc_struct, ipatch, problem_data, pressure, old_pressure,
+                                      ke_der, kw_der, kn_der, ks_der,
+                                      kens_der, kwns_der, knns_der, ksns_der, NULL, NULL, CALCDER));
+                }
+              }
+              break;
+
+            default:
+              break;
+        }
+      })
+                           ); /* End OverlandBC */
 
       ForPatchCellsPerFace(SeepageFaceBC,
                            BeforeAllCells({ vol = dx * dy * dz; }),
                            LoopVars(i, j, k, ival, bc_struct, ipatch, is),
-                           Locals(int ip, im;),
+                           Locals(int ip, im; ),
                            CellSetup(DoNothing),
-                           FACE(LeftFace,  DoNothing),
+                           FACE(LeftFace, DoNothing),
                            FACE(RightFace, DoNothing),
-                           FACE(DownFace,  DoNothing),
-                           FACE(UpFace,    DoNothing),
-                           FACE(BackFace,  DoNothing),
+                           FACE(DownFace, DoNothing),
+                           FACE(UpFace, DoNothing),
+                           FACE(BackFace, DoNothing),
                            FACE(FrontFace,
-                           {
-                             ip = SubvectorEltIndex(p_sub, i, j, k);
-                             im = SubmatrixEltIndex(J_sub, i, j, k);
-
-                             if ((pp[ip]) >= 0.0)
-                             {
-                               cp[im] += (vol / dz) * dt * (1.0 + 0.0);                       //@RMM
-//                  printf("Jac SF: CP=%f im=%d  \n", cp[im], im);
-                             }
-                             else
-                             {
-                               cp[im] += 0.0;
-                             }
-                           }),
+      {
+        ip = SubvectorEltIndex(p_sub, i, j, k);
+        im = SubmatrixEltIndex(J_sub, i, j, k);
+
+        if ((pp[ip]) >= 0.0)
+        {
+          cp[im] += (vol / dz) * dt * (1.0 + 0.0);                                            //RMM
+        }
+        else
+        {
+          cp[im] += 0.0;
+        }
+      }),
                            CellFinalize(DoNothing),
                            AfterAllCells(DoNothing)
-        ); /* End SeepageFaceBC */
+                           ); /* End SeepageFaceBC */
 
       ForPatchCellsPerFace(OverlandKinematicBC,
                            BeforeAllCells(DoNothing),
                            LoopVars(i, j, k, ival, bc_struct, ipatch, is),
                            Locals(int im, ip;
-                                  double *op;),
+                                  double *op; ),
                            CellSetup({ im = SubmatrixEltIndex(J_sub, i, j, k); }),
-                           FACE(LeftFace,  { op = wp; }),
+                           FACE(LeftFace, { op = wp; }),
                            FACE(RightFace, { op = ep; }),
-                           FACE(DownFace,  { op = sop; }),
-                           FACE(UpFace,    { op = np; }),
-                           FACE(BackFace,  { op = lp; }),
+                           FACE(DownFace, { op = sop; }),
+                           FACE(UpFace, { op = np; }),
+                           FACE(BackFace, { op = lp; }),
                            FACE(FrontFace, {
-                               op = up;
-                               /* check if overland flow kicks in */
-                               if (!ovlnd_flag[0])
-                               {
-                                 ip = SubvectorEltIndex(p_sub, i, j, k);
-                                 if ((pp[ip]) > 0.0)
-                                 {
-                                   ovlnd_flag[0] = 1;
-                                 }
-                               }
-                             }),
+        op = up;
+        /* check if overland flow kicks in */
+        if (!ovlnd_flag[0])
+        {
+          ip = SubvectorEltIndex(p_sub, i, j, k);
+          if ((pp[ip]) > 0.0)
+          {
+            ovlnd_flag[0] = 1;
+          }
+        }
+      }),
                            CellFinalize({
-                               cp[im] += op[im];
-                               op[im] = 0.0;       //zero out entry in row of Jacobian
-                             }),
+        cp[im] += op[im];
+        op[im] = 0.0;                              //zero out entry in row of Jacobian
+      }),
                            AfterAllCells(
-                           {
-                             PFModuleInvokeType(OverlandFlowEvalKinInvoke, overlandflow_module_kin,
-                                                (grid, is, bc_struct, ipatch, problem_data, pressure,
-                                                 ke_der, kw_der, kn_der, ks_der,
-                                                 NULL, NULL, NULL, NULL, NULL, NULL, CALCDER));
-                           })
-        ); /* End OverlandKinematicBC */
+      {
+        PFModuleInvokeType(OverlandFlowEvalKinInvoke, overlandflow_module_kin,
+                           (grid, is, bc_struct, ipatch, problem_data, pressure,
+                            ke_der, kw_der, kn_der, ks_der,
+                            NULL, NULL, NULL, NULL, NULL, NULL, CALCDER));
+      })
+                           ); /* End OverlandKinematicBC */
 
       ForPatchCellsPerFace(OverlandDiffusiveBC,
                            BeforeAllCells(DoNothing),
                            LoopVars(i, j, k, ival, bc_struct, ipatch, is),
                            Locals(int im, ip;
-                                  double *op;),
+                                  double *op; ),
                            CellSetup({ im = SubmatrixEltIndex(J_sub, i, j, k); }),
                            FACE(LeftFace, { op = wp; }),
                            FACE(RightFace, { op = ep; }),
@@ -1394,30 +1398,29 @@ void    RichardsJacobianEval(
                            FACE(UpFace, { op = np; }),
                            FACE(BackFace, { op = lp; }),
                            FACE(FrontFace, {
-                               op = up;
-                               /* check if overland flow kicks in */
-                               if (!ovlnd_flag[0])
-                               {
-                                 ip = SubvectorEltIndex(p_sub, i, j, k);
-                                 if ((pp[ip]) > 0.0)
-                                 {
-                                   ovlnd_flag[0] = 1;
-                                 }
-                               }
-                             }),
+        op = up;
+        /* check if overland flow kicks in */
+        if (!ovlnd_flag[0])
+        {
+          ip = SubvectorEltIndex(p_sub, i, j, k);
+          if ((pp[ip]) > 0.0)
+          {
+            ovlnd_flag[0] = 1;
+          }
+        }
+      }),
                            CellFinalize({
-                               cp[im] += op[im];
-                               op[im] = 0.0;       //zero out entry in row of Jacobian
-                             }),
+        cp[im] += op[im];
+        op[im] = 0.0;                              //zero out entry in row of Jacobian
+      }),
                            AfterAllCells(
-                           {
-                             PFModuleInvokeType(OverlandFlowEvalDiffInvoke, overlandflow_module_diff,
-                                                (grid, is, bc_struct, ipatch, problem_data, pressure, old_pressure,
-                                                 ke_der, kw_der, kn_der, ks_der,
-                                                 kens_der, kwns_der, knns_der, ksns_der, NULL, NULL, CALCDER));
-                           })
-        ); /* End OverlandDiffusiveBC */
-
+      {
+        PFModuleInvokeType(OverlandFlowEvalDiffInvoke, overlandflow_module_diff,
+                           (grid, is, bc_struct, ipatch, problem_data, pressure, old_pressure,
+                            ke_der, kw_der, kn_der, ks_der,
+                            kens_der, kwns_der, knns_der, ksns_der, NULL, NULL, CALCDER));
+      })
+                           ); /* End OverlandDiffusiveBC */
     } /* End ipatch loop */
   }            /* End subgrid loop */
 
@@ -1426,7 +1429,7 @@ void    RichardsJacobianEval(
 
 
 
-  if (public_xtra->type == overland_flow)
+  if (public_xtra->type == overland_flow || public_xtra->using_MGSemi == 1)
   {
     // SGS always have to do communication here since
     // each processor may/may not be doing overland flow.
@@ -1463,8 +1466,8 @@ void    RichardsJacobianEval(
     FinalizeVectorUpdate(vector_update_handle);
   }
 
-  /* Build submatrix JC if overland flow case */
-  if (ovlnd_flag[0] && public_xtra->type == overland_flow)
+  /* Build submatrix JC if overland flow case and *not* MGSemi*/
+  if (ovlnd_flag[0] && public_xtra->type == overland_flow && public_xtra->using_MGSemi != 1)
   {
     /* begin loop to build JC */
     ForSubgridI(is, GridSubgrids(grid))
@@ -1544,298 +1547,524 @@ void    RichardsJacobianEval(
         ForPatchCellsPerFace(OverlandKinematicBC,
                              BeforeAllCells(DoNothing),
                              LoopVars(i, j, k, ival, bc_struct, ipatch, is),
-                             Locals(int io, io1, itop, ip, im, k1;),
+                             Locals(int io, io1, itop, ip, im, k1; ),
                              CellSetup(DoNothing),
-                             FACE(LeftFace,  DoNothing), FACE(RightFace, DoNothing),
-                             FACE(DownFace,  DoNothing), FACE(UpFace,    DoNothing),
-                             FACE(BackFace,  DoNothing),
+                             FACE(LeftFace, DoNothing), FACE(RightFace, DoNothing),
+                             FACE(DownFace, DoNothing), FACE(UpFace, DoNothing),
+                             FACE(BackFace, DoNothing),
                              FACE(FrontFace,
-                             {
-                               /* Loop over boundary patches to build JC matrix. */
-                               io = SubmatrixEltIndex(J_sub, i, j, iz);
-                               io1 = SubvectorEltIndex(sx_sub, i, j, 0);
-                               itop = SubvectorEltIndex(top_sub, i, j, 0);
-
-                               /* Update JC */
-                               ip = SubvectorEltIndex(p_sub, i, j, k);
-                               im = SubmatrixEltIndex(J_sub, i, j, k);
-
-                               /* First put contributions from subsurface diagonal onto diagonal of JC */
-                               cp_c[io] = cp[im];
-                               cp[im] = 0.0;         // update JB
-                               /* Now check off-diagonal nodes to see if any surface-surface connections exist */
-                               /* West */
-                               k1 = (int)top_dat[itop - 1];
-
-                               if (k1 >= 0)
-                               {
-                                 if (k1 == k)         /*west node is also surface node */
-                                 {
-                                   wp_c[io] += wp[im];
-                                   wp[im] = 0.0;           // update JB
-                                 }
-                               }
-                               /* East */
-                               k1 = (int)top_dat[itop + 1];
-                               if (k1 >= 0)
-                               {
-                                 if (k1 == k)         /*east node is also surface node */
-                                 {
-                                   ep_c[io] += ep[im];
-                                   ep[im] = 0.0;           //update JB
-                                 }
-                               }
-                               /* South */
-                               k1 = (int)top_dat[itop - sy_v];
-                               if (k1 >= 0)
-                               {
-                                 if (k1 == k)         /*south node is also surface node */
-                                 {
-                                   sop_c[io] += sop[im];
-                                   sop[im] = 0.0;           //update JB
-                                 }
-                               }
-                               /* North */
-                               k1 = (int)top_dat[itop + sy_v];
-                               if (k1 >= 0)
-                               {
-                                 if (k1 == k)         /*north node is also surface node */
-                                 {
-                                   np_c[io] += np[im];
-                                   np[im] = 0.0;           // Update JB
-                                 }
-                               }
-
-                               /* Now add overland contributions to JC */
-                               if ((pp[ip]) > 0.0)
-                               {
-                                 /*diagonal term */
-                                 cp_c[io] += (vol / dz) + (vol / ffy) * dt * (ke_der[io1] - kw_der[io1])
-                                             + (vol / ffx) * dt * (kn_der[io1] - ks_der[io1]);
-                               }
-
-                               /*west term */
-                               wp_c[io] -= (vol / ffy) * dt * (ke_der[io1 - 1]);
-
-                               /*East term */
-                               ep_c[io] += (vol / ffy) * dt * (kw_der[io1 + 1]);
-
-                               /*south term */
-                               sop_c[io] -= (vol / ffx) * dt * (kn_der[io1 - sy_v]);
-
-                               /*north term */
-                               np_c[io] += (vol / ffx) * dt * (ks_der[io1 + sy_v]);
-                             }),
+        {
+          /* Loop over boundary patches to build JC matrix. */
+          io = SubmatrixEltIndex(J_sub, i, j, iz);
+          io1 = SubvectorEltIndex(sx_sub, i, j, 0);
+          itop = SubvectorEltIndex(top_sub, i, j, 0);
+
+          /* Update JC */
+          ip = SubvectorEltIndex(p_sub, i, j, k);
+          im = SubmatrixEltIndex(J_sub, i, j, k);
+
+          /* First put contributions from subsurface diagonal onto diagonal of JC */
+          cp_c[io] = cp[im];
+          cp[im] = 0.0;                              // update JB
+          /* Now check off-diagonal nodes to see if any surface-surface connections exist */
+          /* West */
+          k1 = (int)top_dat[itop - 1];
+
+          if (k1 >= 0)
+          {
+            if (k1 == k)                              /*west node is also surface node */
+            {
+              wp_c[io] += wp[im];
+              wp[im] = 0.0;                                // update JB
+            }
+          }
+          /* East */
+          k1 = (int)top_dat[itop + 1];
+          if (k1 >= 0)
+          {
+            if (k1 == k)                              /*east node is also surface node */
+            {
+              ep_c[io] += ep[im];
+              ep[im] = 0.0;                                //update JB
+            }
+          }
+          /* South */
+          k1 = (int)top_dat[itop - sy_v];
+          if (k1 >= 0)
+          {
+            if (k1 == k)                              /*south node is also surface node */
+            {
+              sop_c[io] += sop[im];
+              sop[im] = 0.0;                                //update JB
+            }
+          }
+          /* North */
+          k1 = (int)top_dat[itop + sy_v];
+          if (k1 >= 0)
+          {
+            if (k1 == k)                              /*north node is also surface node */
+            {
+              np_c[io] += np[im];
+              np[im] = 0.0;                                // Update JB
+            }
+          }
+
+          /* Now add overland contributions to JC */
+          if ((pp[ip]) > 0.0)
+          {
+            /*diagonal term */
+            cp_c[io] += (vol / dz) + (vol / ffy) * dt * (ke_der[io1] - kw_der[io1])
+                        + (vol / ffx) * dt * (kn_der[io1] - ks_der[io1]);
+          }
+
+          /*west term */
+          wp_c[io] -= (vol / ffy) * dt * (ke_der[io1 - 1]);
+
+          /*East term */
+          ep_c[io] += (vol / ffy) * dt * (kw_der[io1 + 1]);
+
+          /*south term */
+          sop_c[io] -= (vol / ffx) * dt * (kn_der[io1 - sy_v]);
+
+          /*north term */
+          np_c[io] += (vol / ffx) * dt * (ks_der[io1 + sy_v]);
+        }),
                              CellFinalize(DoNothing),
                              AfterAllCells(DoNothing)
-          ); /* End OverlandKinematicBC */
+                             ); /* End OverlandKinematicBC */
 
         ForPatchCellsPerFace(OverlandDiffusiveBC,
                              BeforeAllCells(DoNothing),
                              LoopVars(i, j, k, ival, bc_struct, ipatch, is),
-                             Locals(int io, io1, itop, ip, im, k1;),
+                             Locals(int io, io1, itop, ip, im, k1; ),
                              CellSetup(DoNothing),
-                             FACE(LeftFace,  DoNothing), FACE(RightFace, DoNothing),
-                             FACE(DownFace,  DoNothing), FACE(UpFace,    DoNothing),
-                             FACE(BackFace,  DoNothing),
+                             FACE(LeftFace, DoNothing), FACE(RightFace, DoNothing),
+                             FACE(DownFace, DoNothing), FACE(UpFace, DoNothing),
+                             FACE(BackFace, DoNothing),
                              FACE(FrontFace,
-                             {
-
-                               /* Loop over boundary patches to build JC matrix.
-                                */
-                               io = SubmatrixEltIndex(J_sub, i, j, iz);
-                               io1 = SubvectorEltIndex(sx_sub, i, j, 0);
-                               itop = SubvectorEltIndex(top_sub, i, j, 0);
-
-                               /* Update JC */
-                               ip = SubvectorEltIndex(p_sub, i, j, k);
-                               im = SubmatrixEltIndex(J_sub, i, j, k);
-
-                               /* First put contributions from subsurface diagonal onto diagonal of JC */
-                               cp_c[io] = cp[im];
-                               cp[im] = 0.0;         // update JB
-                               /* Now check off-diagonal nodes to see if any surface-surface connections exist */
-                               /* West */
-                               k1 = (int)top_dat[itop - 1];
-
-                               if (k1 >= 0)
-                               {
-                                 if (k1 == k)         /*west node is also surface node */
-                                 {
-                                   wp_c[io] += wp[im];
-                                   wp[im] = 0.0;           // update JB
-                                 }
-                               }
-                               /* East */
-                               k1 = (int)top_dat[itop + 1];
-                               if (k1 >= 0)
-                               {
-                                 if (k1 == k)         /*east node is also surface node */
-                                 {
-                                   ep_c[io] += ep[im];
-                                   ep[im] = 0.0;           //update JB
-                                 }
-                               }
-                               /* South */
-                               k1 = (int)top_dat[itop - sy_v];
-                               if (k1 >= 0)
-                               {
-                                 if (k1 == k)         /*south node is also surface node */
-                                 {
-                                   sop_c[io] += sop[im];
-                                   sop[im] = 0.0;           //update JB
-                                 }
-                               }
-                               /* North */
-                               k1 = (int)top_dat[itop + sy_v];
-                               if (k1 >= 0)
-                               {
-                                 if (k1 == k)         /*north node is also surface node */
-                                 {
-                                   np_c[io] += np[im];
-                                   np[im] = 0.0;           // Update JB
-                                 }
-                               }
-
-                               /* Now add overland contributions to JC */
-                               if ((pp[ip]) > 0.0)
-                               {
-                                 /*diagonal term */
-                                 cp_c[io] += (vol / dz) + (vol / ffy) * dt * (ke_der[io1] - kw_der[io1])
-                                             + (vol / ffx) * dt * (kn_der[io1] - ks_der[io1]);
-                               }
-                               /*west term */
-                               wp_c[io] -= (vol / ffy) * dt * (kwns_der[io1]);
-
-                               /*East term */
-                               ep_c[io] += (vol / ffy) * dt * (kens_der[io1]);
-
-                               /*south term */
-                               sop_c[io] -= (vol / ffx) * dt * (ksns_der[io1]);
-
-                               /*north term */
-                               np_c[io] += (vol / ffx) * dt * (knns_der[io1]);
-                             }),
+        {
+          /* Loop over boundary patches to build JC matrix.
+           */
+          io = SubmatrixEltIndex(J_sub, i, j, iz);
+          io1 = SubvectorEltIndex(sx_sub, i, j, 0);
+          itop = SubvectorEltIndex(top_sub, i, j, 0);
+
+          /* Update JC */
+          ip = SubvectorEltIndex(p_sub, i, j, k);
+          im = SubmatrixEltIndex(J_sub, i, j, k);
+
+          /* First put contributions from subsurface diagonal onto diagonal of JC */
+          cp_c[io] = cp[im];
+          cp[im] = 0.0;                              // update JB
+          /* Now check off-diagonal nodes to see if any surface-surface connections exist */
+          /* West */
+          k1 = (int)top_dat[itop - 1];
+
+          if (k1 >= 0)
+          {
+            if (k1 == k)                              /*west node is also surface node */
+            {
+              wp_c[io] += wp[im];
+              wp[im] = 0.0;                                // update JB
+            }
+          }
+          /* East */
+          k1 = (int)top_dat[itop + 1];
+          if (k1 >= 0)
+          {
+            if (k1 == k)                              /*east node is also surface node */
+            {
+              ep_c[io] += ep[im];
+              ep[im] = 0.0;                                //update JB
+            }
+          }
+          /* South */
+          k1 = (int)top_dat[itop - sy_v];
+          if (k1 >= 0)
+          {
+            if (k1 == k)                              /*south node is also surface node */
+            {
+              sop_c[io] += sop[im];
+              sop[im] = 0.0;                                //update JB
+            }
+          }
+          /* North */
+          k1 = (int)top_dat[itop + sy_v];
+          if (k1 >= 0)
+          {
+            if (k1 == k)                              /*north node is also surface node */
+            {
+              np_c[io] += np[im];
+              np[im] = 0.0;                                // Update JB
+            }
+          }
+
+          /* Now add overland contributions to JC */
+          if ((pp[ip]) > 0.0)
+          {
+            /*diagonal term */
+            cp_c[io] += (vol / dz) + (vol / ffy) * dt * (ke_der[io1] - kw_der[io1])
+                        + (vol / ffx) * dt * (kn_der[io1] - ks_der[io1]);
+          }
+          /*west term */
+          wp_c[io] -= (vol / ffy) * dt * (kwns_der[io1]);
+
+          /*East term */
+          ep_c[io] += (vol / ffy) * dt * (kens_der[io1]);
+
+          /*south term */
+          sop_c[io] -= (vol / ffx) * dt * (ksns_der[io1]);
+
+          /*north term */
+          np_c[io] += (vol / ffx) * dt * (knns_der[io1]);
+        }),
                              CellFinalize(DoNothing),
                              AfterAllCells(DoNothing)
-          ); /* End OverlandDiffusiveBC */
+                             ); /* End OverlandDiffusiveBC */
 
         ForPatchCellsPerFace(OverlandBC,
                              BeforeAllCells(DoNothing),
                              LoopVars(i, j, k, ival, bc_struct, ipatch, is),
-                             Locals(int io, io1, itop, ip, im, k1;),
+                             Locals(int io, io1, itop, ip, im, k1; ),
                              CellSetup(DoNothing),
-                             FACE(LeftFace,  DoNothing), FACE(RightFace, DoNothing),
-                             FACE(DownFace,  DoNothing), FACE(UpFace,    DoNothing),
-                             FACE(BackFace,  DoNothing),
+                             FACE(LeftFace, DoNothing), FACE(RightFace, DoNothing),
+                             FACE(DownFace, DoNothing), FACE(UpFace, DoNothing),
+                             FACE(BackFace, DoNothing),
                              FACE(FrontFace,
-                             {
-                               /* Loop over boundary patches to build JC matrix.
-                                */
-                               io = SubmatrixEltIndex(J_sub, i, j, iz);
-                               io1 = SubvectorEltIndex(sx_sub, i, j, 0);
-                               itop = SubvectorEltIndex(top_sub, i, j, 0);
-
-                               /* Update JC */
-                               ip = SubvectorEltIndex(p_sub, i, j, k);
-                               im = SubmatrixEltIndex(J_sub, i, j, k);
-
-                               /* First put contributions from subsurface diagonal onto diagonal of JC */
-                               cp_c[io] = cp[im];
-                               cp[im] = 0.0;         // update JB
-                               /* Now check off-diagonal nodes to see if any surface-surface connections exist */
-                               /* West */
-                               k1 = (int)top_dat[itop - 1];
-
-                               if (k1 >= 0)
-                               {
-                                 if (k1 == k)         /*west node is also surface node */
-                                 {
-                                   wp_c[io] += wp[im];
-                                   wp[im] = 0.0;           // update JB
-                                 }
-                               }
-                               /* East */
-                               k1 = (int)top_dat[itop + 1];
-                               if (k1 >= 0)
-                               {
-                                 if (k1 == k)         /*east node is also surface node */
-                                 {
-                                   ep_c[io] += ep[im];
-                                   ep[im] = 0.0;           //update JB
-                                 }
-                               }
-                               /* South */
-                               k1 = (int)top_dat[itop - sy_v];
-                               if (k1 >= 0)
-                               {
-                                 if (k1 == k)         /*south node is also surface node */
-                                 {
-                                   sop_c[io] += sop[im];
-                                   sop[im] = 0.0;           //update JB
-                                 }
-                               }
-                               /* North */
-                               k1 = (int)top_dat[itop + sy_v];
-                               if (k1 >= 0)
-                               {
-                                 if (k1 == k)         /*north node is also surface node */
-                                 {
-                                   np_c[io] += np[im];
-                                   np[im] = 0.0;           // Update JB
-                                 }
-                               }
-
-                               /* Now add overland contributions to JC */
-                               if ((pp[ip]) > 0.0)
-                               {
-                                 /*diagonal term */
-                                 cp_c[io] += (vol / dz) + (vol / ffy) * dt * (ke_der[io1] - kw_der[io1])
-                                             + (vol / ffx) * dt * (kn_der[io1] - ks_der[io1]);
-                               }
-                               else
-                               {
-                                 // Laura's version
-                                 cp_c[io] += 0.0 + dt * (vol / dz) * (public_xtra->SpinupDampP1 * exp(pfmin(pp[ip], 0.0) * public_xtra->SpinupDampP1) * public_xtra->SpinupDampP2); //NBE
-                               }
-
-                               if (diffusive == 0)
-                               {
-                                 /*west term */
-                                 wp_c[io] -= (vol / ffy) * dt * (ke_der[io1 - 1]);
-
-                                 /*East term */
-                                 ep_c[io] += (vol / ffy) * dt * (kw_der[io1 + 1]);
-
-                                 /*south term */
-                                 sop_c[io] -= (vol / ffx) * dt * (kn_der[io1 - sy_v]);
-
-                                 /*north term */
-                                 np_c[io] += (vol / ffx) * dt * (ks_der[io1 + sy_v]);
-                               }
-                               else
-                               {
-                                 /*west term */
-                                 wp_c[io] -= (vol / ffy) * dt * (kwns_der[io1]);
-
-                                 /*East term */
-                                 ep_c[io] += (vol / ffy) * dt * (kens_der[io1]);
-
-                                 /*south term */
-                                 sop_c[io] -= (vol / ffx) * dt * (ksns_der[io1]);
-
-                                 /*north term */
-                                 np_c[io] += (vol / ffx) * dt * (knns_der[io1]);
-                               }
-                             }),
+        {
+          /* Loop over boundary patches to build JC matrix.
+           */
+          io = SubmatrixEltIndex(J_sub, i, j, iz);
+          io1 = SubvectorEltIndex(sx_sub, i, j, 0);
+          itop = SubvectorEltIndex(top_sub, i, j, 0);
+
+          /* Update JC */
+          ip = SubvectorEltIndex(p_sub, i, j, k);
+          im = SubmatrixEltIndex(J_sub, i, j, k);
+
+          /* First put contributions from subsurface diagonal onto diagonal of JC */
+          cp_c[io] = cp[im];
+          cp[im] = 0.0;                              // update JB
+          /* Now check off-diagonal nodes to see if any surface-surface connections exist */
+          /* West */
+          k1 = (int)top_dat[itop - 1];
+
+          if (k1 >= 0)
+          {
+            if (k1 == k)                              /*west node is also surface node */
+            {
+              wp_c[io] += wp[im];
+              wp[im] = 0.0;                                // update JB
+            }
+          }
+          /* East */
+          k1 = (int)top_dat[itop + 1];
+          if (k1 >= 0)
+          {
+            if (k1 == k)                              /*east node is also surface node */
+            {
+              ep_c[io] += ep[im];
+              ep[im] = 0.0;                                //update JB
+            }
+          }
+          /* South */
+          k1 = (int)top_dat[itop - sy_v];
+          if (k1 >= 0)
+          {
+            if (k1 == k)                              /*south node is also surface node */
+            {
+              sop_c[io] += sop[im];
+              sop[im] = 0.0;                                //update JB
+            }
+          }
+          /* North */
+          k1 = (int)top_dat[itop + sy_v];
+          if (k1 >= 0)
+          {
+            if (k1 == k)                              /*north node is also surface node */
+            {
+              np_c[io] += np[im];
+              np[im] = 0.0;                                // Update JB
+            }
+          }
+
+          /* Now add overland contributions to JC */
+          if ((pp[ip]) > 0.0)
+          {
+            /*diagonal term */
+            cp_c[io] += (vol / dz) + (vol / ffy) * dt * (ke_der[io1] - kw_der[io1])
+                        + (vol / ffx) * dt * (kn_der[io1] - ks_der[io1]);
+          }
+          else
+          {
+            // Laura's version
+            cp_c[io] += 0.0 + dt * (vol / dz) * (public_xtra->SpinupDampP1 * exp(pfmin(pp[ip], 0.0) * public_xtra->SpinupDampP1) * public_xtra->SpinupDampP2);                      //NBE
+          }
+
+          if (diffusive == 0)
+          {
+            /*west term */
+            wp_c[io] -= (vol / ffy) * dt * (ke_der[io1 - 1]);
+
+            /*East term */
+            ep_c[io] += (vol / ffy) * dt * (kw_der[io1 + 1]);
+
+            /*south term */
+            sop_c[io] -= (vol / ffx) * dt * (kn_der[io1 - sy_v]);
+
+            /*north term */
+            np_c[io] += (vol / ffx) * dt * (ks_der[io1 + sy_v]);
+          }
+          else
+          {
+            /*west term */
+            wp_c[io] -= (vol / ffy) * dt * (kwns_der[io1]);
+
+            /*East term */
+            ep_c[io] += (vol / ffy) * dt * (kens_der[io1]);
+
+            /*south term */
+            sop_c[io] -= (vol / ffx) * dt * (ksns_der[io1]);
+
+            /*north term */
+            np_c[io] += (vol / ffx) * dt * (knns_der[io1]);
+          }
+        }),
                              CellFinalize(DoNothing),
                              AfterAllCells(DoNothing)
-          ); /* End OverlandBC */
+                             ); /* End OverlandBC */
       } /* End ipatch loop */
     }             /* End subgrid loop */
   }
 
+/* MGSemi Jacobian construction; this should (will) only be active if we have
+ * one of the overland flow cases (KWE, DWE, Overland) and MGSemi chosen as preconditioner.
+ * This is set by the overland_flow variable.  This logic is similar
+ * to the case statements in OverlandKinematic above
+ * for PFMG/SMG/PFMGOctree(case overland_flow) which are split into the surface and
+ * subsurface parts (RMM)*/
+
+/* The MGSemi center part this is  only be active if we have
+ * and MGSemi chosen as preconditioner.  This logic is similar
+ * to, and was patterned after, the case statements in OverlandFlow above
+ * where the FD Jacobian (case no_nonlinear), MGSemi (case simple) and PFMG
+ * (case overland_flow) are enumerated explicitly (RMM)*/
+
+  if (public_xtra->using_MGSemi == 1)
+  {
+    /* begin loop to build J for MGSemi */
+    ForSubgridI(is, GridSubgrids(grid))
+    {
+      subgrid = GridSubgrid(grid, is);
+
+      dx = SubgridDX(subgrid);
+      dy = SubgridDY(subgrid);
+      dz = SubgridDZ(subgrid);
+
+      double vol = dx * dy * dz;
+
+      ffx = dy * dz;
+      ffy = dx * dz;
+      ffz = dx * dy;
+
+      p_sub = VectorSubvector(pressure, is);
+
+      J_sub = MatrixSubmatrix(J, is);
+
+      kw_sub = VectorSubvector(KW, is);
+      ke_sub = VectorSubvector(KE, is);
+      kn_sub = VectorSubvector(KN, is);
+      ks_sub = VectorSubvector(KS, is);
+      kwns_sub = VectorSubvector(KWns, is);
+      kens_sub = VectorSubvector(KEns, is);
+      knns_sub = VectorSubvector(KNns, is);
+      ksns_sub = VectorSubvector(KSns, is);
+
+      top_sub = VectorSubvector(top, is);
+      sx_sub = VectorSubvector(slope_x, is);
+
+      sy_v = SubvectorNX(sx_sub);
+      nx_m = SubmatrixNX(J_sub);
+      ny_m = SubmatrixNY(J_sub);
+      sy_m = nx_m;
+      sz_m = nx_m * ny_m;
+
+      ix = SubgridIX(subgrid);
+      iy = SubgridIY(subgrid);
+      iz = SubgridIZ(subgrid);
+
+      nx = SubgridNX(subgrid);
+      ny = SubgridNY(subgrid);
+
+      pp = SubvectorData(p_sub);
+      /* for Bmat */
+      cp = SubmatrixStencilData(J_sub, 0);
+      wp = SubmatrixStencilData(J_sub, 1);
+      ep = SubmatrixStencilData(J_sub, 2);
+      sop = SubmatrixStencilData(J_sub, 3);
+      np = SubmatrixStencilData(J_sub, 4);
+      lp = SubmatrixStencilData(J_sub, 5);
+      up = SubmatrixStencilData(J_sub, 6);
+
+      kw_der = SubvectorData(kw_sub);
+      ke_der = SubvectorData(ke_sub);
+      kn_der = SubvectorData(kn_sub);
+      ks_der = SubvectorData(ks_sub);
+      kwns_der = SubvectorData(kwns_sub);
+      kens_der = SubvectorData(kens_sub);
+      knns_der = SubvectorData(knns_sub);
+      ksns_der = SubvectorData(ksns_sub);
+
+      top_dat = SubvectorData(top_sub);
+
+      ForBCStructNumPatches(ipatch, bc_struct)
+      {
+        ForPatchCellsPerFace(OverlandKinematicBC,
+                             BeforeAllCells(DoNothing),
+                             LoopVars(i, j, k, ival, bc_struct, ipatch, is),
+                             Locals(int io1, ip, im; ),
+                             CellSetup(DoNothing),
+                             FACE(LeftFace, DoNothing), FACE(RightFace, DoNothing),
+                             FACE(DownFace, DoNothing), FACE(UpFace, DoNothing),
+                             FACE(BackFace, DoNothing),
+                             FACE(FrontFace,
+        {
+          /* Loop over boundary patches to build J matrix. */
+          io1 = SubvectorEltIndex(sx_sub, i, j, 0);
+
+          /* Update J */
+          ip = SubvectorEltIndex(p_sub, i, j, k);
+          im = SubmatrixEltIndex(J_sub, i, j, k);
+
+          /* Now add overland contributions to J similar to JC above */
+          if ((pp[ip]) > 0.0)
+          {
+            /*diagonal term */
+            cp[im] += (vol / dz) + (vol / ffy) * dt * (ke_der[io1] - kw_der[io1])
+                      + (vol / ffx) * dt * (kn_der[io1] - ks_der[io1]);
+          }
+
+          /*west term */
+          wp[im] -= (vol / ffy) * dt * (ke_der[io1 - 1]);
+
+          /*East term */
+          ep[im] += (vol / ffy) * dt * (kw_der[io1 + 1]);
+
+          /*south term */
+          sop[im] -= (vol / ffx) * dt * (kn_der[io1 - sy_v]);
+
+          /*north term */
+          np[im] += (vol / ffx) * dt * (ks_der[io1 + sy_v]);
+        }),
+                             CellFinalize(DoNothing),
+                             AfterAllCells(DoNothing)
+                             ); /* End OverlandKinematicBC */
+
+        ForPatchCellsPerFace(OverlandDiffusiveBC,
+                             BeforeAllCells(DoNothing),
+                             LoopVars(i, j, k, ival, bc_struct, ipatch, is),
+                             Locals(int io1, ip, im; ),
+                             CellSetup(DoNothing),
+                             FACE(LeftFace, DoNothing), FACE(RightFace, DoNothing),
+                             FACE(DownFace, DoNothing), FACE(UpFace, DoNothing),
+                             FACE(BackFace, DoNothing),
+                             FACE(FrontFace,
+        {
+          /* Loop over boundary patches to build J matrix.
+           */
+          io1 = SubvectorEltIndex(sx_sub, i, j, 0);
+          ip = SubvectorEltIndex(p_sub, i, j, k);
+          im = SubmatrixEltIndex(J_sub, i, j, k);
+
+          /* Now add overland contributions */
+          if ((pp[ip]) > 0.0)
+          {
+            /*diagonal term */
+            cp[im] += (vol / dz) + (vol / ffy) * dt * (ke_der[io1] - kw_der[io1])
+                      + (vol / ffx) * dt * (kn_der[io1] - ks_der[io1]);
+          }
+          /*west term */
+          wp[im] -= (vol / ffy) * dt * (kwns_der[io1]);
+
+          /*East term */
+          ep[im] += (vol / ffy) * dt * (kens_der[io1]);
+
+          /*south term */
+          sop[im] -= (vol / ffx) * dt * (ksns_der[io1]);
+
+          /*north term */
+          np[im] += (vol / ffx) * dt * (knns_der[io1]);
+        }),
+                             CellFinalize(DoNothing),
+                             AfterAllCells(DoNothing)
+                             ); /* End OverlandDiffusiveBC */
 
+        ForPatchCellsPerFace(OverlandBC,
+                             BeforeAllCells(DoNothing),
+                             LoopVars(i, j, k, ival, bc_struct, ipatch, is),
+                             Locals(int io1, ip, im; ),
+                             CellSetup(DoNothing),
+                             FACE(LeftFace, DoNothing), FACE(RightFace, DoNothing),
+                             FACE(DownFace, DoNothing), FACE(UpFace, DoNothing),
+                             FACE(BackFace, DoNothing),
+                             FACE(FrontFace,
+        {
+          /* Loop over boundary patches to build J matrix.
+           */
+          io1 = SubvectorEltIndex(sx_sub, i, j, 0);
+          ip = SubvectorEltIndex(p_sub, i, j, k);
+          im = SubmatrixEltIndex(J_sub, i, j, k);
+
+          /* Now add overland contributions to the jacobian */
+          if ((pp[ip]) > 0.0)
+          {
+            /*diagonal term */
+            /* original preconditioner from KM2006 here for historical reasons; performance is much slower than newer formulation below */
+            //cp[im] += (vol * z_mult_dat[ip]) / (dz * Mean(z_mult_dat[ip], z_mult_dat[ip + sz_v])) * (dt + 1);
+            cp[im] += (vol / dz) + (vol / ffy) * dt * (ke_der[io1] - kw_der[io1])
+                      + (vol / ffx) * dt * (kn_der[io1] - ks_der[io1]);
+          }
+          else
+          {
+            // Laura's version; old diffusive wave formulation which is now replaced by DWE
+            cp[im] += 0.0 + dt * (vol / dz) * (public_xtra->SpinupDampP1 * exp(pfmin(pp[ip], 0.0) * public_xtra->SpinupDampP1) * public_xtra->SpinupDampP2);                      //NBE
+          }
+
+          if (diffusive == 0)
+          {
+            /*west term */
+            wp[im] -= (vol / ffy) * dt * (ke_der[io1 - 1]);
+
+            //  /*East term */
+            ep[im] += (vol / ffy) * dt * (kw_der[io1 + 1]);
+
+            //  /*south term */
+            sop[im] -= (vol / ffx) * dt * (kn_der[io1 - sy_v]);
+
+            //  /*north term */
+            np[im] += (vol / ffx) * dt * (ks_der[io1 + sy_v]);
+          }
+          else
+          {
+            /*west term */
+            wp[im] -= (vol / ffy) * dt * (kwns_der[io1]);
+
+            /*East term */
+            ep[im] += (vol / ffy) * dt * (kens_der[io1]);
+
+            /*south term */
+            sop[im] -= (vol / ffx) * dt * (ksns_der[io1]);
+
+            /*north term */
+            np[im] += (vol / ffx) * dt * (knns_der[io1]);
+          }
+        }),
+                             CellFinalize(DoNothing),
+                             AfterAllCells(DoNothing)
+                             ); /* End OverlandBC */
+      } /* End ipatch loop */
+    }             /* End subgrid loop */
+  }
 
   /* Set pressures outside domain to zero.
    * Recall: equation to solve is f = 0, so components of f outside
@@ -2104,6 +2333,7 @@ PFModule   *RichardsJacobianEvalNewPublicXtra(char *name)
   int switch_value;
   NameArray switch_na;
   NameArray upwind_switch_na;
+  NameArray precond_switch_na;
 
 
   (void)name;
@@ -2116,7 +2346,23 @@ PFModule   *RichardsJacobianEvalNewPublicXtra(char *name)
   sprintf(key, "OverlandSpinupDampP2");
   public_xtra->SpinupDampP2 = GetDoubleDefault(key, 0.0);    // NBE
 
-  ///* parameters for upwinding formulation for TFG */
+/* get preconditioner to check for MGSemi to use custom overland flow formulation*/
+  precond_switch_na = NA_NewNameArray("NoPC MGSemi SMG PFMG PFMGOctree");
+  sprintf(key, "Solver.Linear.Preconditioner");
+  switch_name = GetStringDefault(key, "MGSemi");
+  switch_value = NA_NameToIndexExitOnError(precond_switch_na, switch_name, key);
+  if (switch_value == 1)
+  {
+    /* (RMM) Include a variable track if MGSemi=True *and* Jacobian=True */
+    public_xtra->using_MGSemi = 1;
+  }
+  else
+  {
+    public_xtra->using_MGSemi = 0;
+  }
+  NA_FreeNameArray(precond_switch_na);
+
+  /* parameters for upwinding formulation for TFG */
   upwind_switch_na = NA_NewNameArray("Original UpwindSine Upwind");
   sprintf(key, "Solver.TerrainFollowingGrid.SlopeUpwindFormulation");
   switch_name = GetStringDefault(key, "Original");
@@ -2157,6 +2403,8 @@ PFModule   *RichardsJacobianEvalNewPublicXtra(char *name)
     case 0:
     {
       public_xtra->type = no_nonlinear_jacobian;
+      /*(RMM) set back to zero so that cases like MGSemi with FD jacobian (FALSE) are not affected*/
+      public_xtra->using_MGSemi = 0;
       break;
     }
 
diff --git a/pfsimulator/parflow_lib/sadvect.win32.c b/pfsimulator/parflow_lib/sadvect.win32.c
index 9f374ad6b..8f9e3959e 100644
--- a/pfsimulator/parflow_lib/sadvect.win32.c
+++ b/pfsimulator/parflow_lib/sadvect.win32.c
@@ -9,32 +9,32 @@
 
 integer c__1 = 1;
 
-/*BHEADER*********************************************************************
+/*BHEADER**********************************************************************
  *
- * Copyright (c) 1995-2009, Lawrence Livermore National Security,
- * LLC. Produced at the Lawrence Livermore National Laboratory. Written
- * by the Parflow Team (see the CONTRIBUTORS file)
- * <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+ *  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+ *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+ *  by the Parflow Team (see the CONTRIBUTORS file)
+ *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
  *
- * This file is part of Parflow. For details, see
- * http://www.llnl.gov/casc/parflow
+ *  This file is part of Parflow. For details, see
+ *  http://www.llnl.gov/casc/parflow
  *
- * Please read the COPYRIGHT file or Our Notice and the LICENSE file
- * for the GNU Lesser General Public License.
+ *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+ *  for the GNU Lesser General Public License.
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License (as published
- * by the Free Software Foundation) version 2.1 dated February 1999.
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License (as published
+ *  by the Free Software Foundation) version 2.1 dated February 1999.
  *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- * and conditions of the GNU General Public License for more details.
+ *  This program is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+ *  and conditions of the GNU General Public License for more details.
  *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- * USA
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ *  USA
  **********************************************************************EHEADER********************************************************************
  ****/
 /****************************************************************************
@@ -56,7 +56,7 @@ doublereal * s, *sn, *uedge, *vedge, *wedge, *betaedge, *phi, *viscos, *densty,
 *gravty, *slx, *sly, *slz;
 integer *lohi, *dlohi;
 doublereal *hx, *dt, *sbot, *stop, *sbotp, *sfrt, *sbck, *sleft, *sright, *
-  sfluxz, *dxscr, *dyscr, *dzscr, *dzfrm;
+           sfluxz, *dxscr, *dyscr, *dzscr, *dzfrm;
 {
   /* Initialized data */
 
@@ -65,15 +65,15 @@ doublereal *hx, *dt, *sbot, *stop, *sbotp, *sfrt, *sbck, *sleft, *sright, *
 
   /* System generated locals */
   integer s_dim1, s_dim2, s_offset, sn_dim1, sn_dim2, sn_offset, uedge_dim1,
-    uedge_dim2, uedge_offset, vedge_dim1, vedge_dim2, vedge_offset,
-    wedge_dim1, wedge_dim2, wedge_offset, betaedge_dim1,
-    betaedge_dim2, betaedge_offset, phi_dim1, phi_dim2, phi_offset,
-    slx_dim1, slx_offset, sly_dim1, sly_offset, slz_dim1, slz_dim2,
-    slz_offset, sbot_dim1, sbot_offset, stop_dim1, stop_offset,
-    sbotp_dim1, sbotp_offset, sbck_dim1, sbck_offset, sfrt_dim1,
-    sfrt_offset, sleft_offset, sright_offset, sfluxz_offset,
-    dxscr_dim1, dxscr_offset, dyscr_dim1, dyscr_offset, dzscr_dim1,
-    dzscr_offset, dzfrm_dim1, dzfrm_offset, i__1, i__2, i__3;
+          uedge_dim2, uedge_offset, vedge_dim1, vedge_dim2, vedge_offset,
+          wedge_dim1, wedge_dim2, wedge_offset, betaedge_dim1,
+          betaedge_dim2, betaedge_offset, phi_dim1, phi_dim2, phi_offset,
+          slx_dim1, slx_offset, sly_dim1, sly_offset, slz_dim1, slz_dim2,
+          slz_offset, sbot_dim1, sbot_offset, stop_dim1, stop_offset,
+          sbotp_dim1, sbotp_offset, sbck_dim1, sbck_offset, sfrt_dim1,
+          sfrt_offset, sleft_offset, sright_offset, sfluxz_offset,
+          dxscr_dim1, dxscr_offset, dyscr_dim1, dyscr_offset, dzscr_dim1,
+          dzscr_offset, dzfrm_dim1, dzfrm_offset, i__1, i__2, i__3;
   doublereal d__1, d__2, d__3;
 
   /* Local variables */
@@ -88,9 +88,9 @@ doublereal *hx, *dt, *sbot, *stop, *sbotp, *sfrt, *sbck, *sleft, *sright, *
   doublereal dx, dy, dz, phiinv, supw_m__, supw_p__, mu0, mu1;
   extern /* Subroutine */ int rpsolv_();
   doublereal dth, dxh, dyh, dzh, dxi, dyi, dzi, cux, cuy, cuz,
-    thi_xhi__, thi_yhi__, thi_zhi__, sux, suy, suz, tlo_xhi__, den0,
-    den1, tlo_yhi__, tlo_zhi__, thi_xlo__, thi_ylo__, thi_zlo__,
-    tlo_xlo__, tlo_ylo__, tlo_zlo__, mu0i, mu1i;
+             thi_xhi__, thi_yhi__, thi_zhi__, sux, suy, suz, tlo_xhi__, den0,
+             den1, tlo_yhi__, tlo_zhi__, thi_xlo__, thi_ylo__, thi_zlo__,
+             tlo_xlo__, tlo_ylo__, tlo_zlo__, mu0i, mu1i;
   extern /* Subroutine */ int sslopez_();
 
 /*     ::: argument declarations */
@@ -834,8 +834,8 @@ doublereal *dxscr, *dyscr;
 
   /* System generated locals */
   integer s_dim1, s_dim2, s_offset, slx_dim1, slx_offset, sly_dim1,
-    sly_offset, dxscr_dim1, dxscr_offset, dyscr_dim1, dyscr_offset,
-    i__1, i__2;
+          sly_offset, dxscr_dim1, dxscr_offset, dyscr_dim1, dyscr_offset,
+          i__1, i__2;
   doublereal d__1, d__2, d__3;
 
   /* Builtin functions */
@@ -1159,8 +1159,8 @@ doublereal *dzscr, *dzfrm;
 
   /* System generated locals */
   integer s_dim1, s_dim2, s_offset, w_dim1, w_dim2, w_offset, betaedge_dim1,
-    betaedge_dim2, betaedge_offset, slz_dim1, slz_dim2, slz_offset,
-    dzscr_dim1, dzscr_offset, dzfrm_dim1, dzfrm_offset, i__1, i__2;
+          betaedge_dim2, betaedge_offset, slz_dim1, slz_dim2, slz_offset,
+          dzscr_dim1, dzscr_offset, dzfrm_dim1, dzfrm_offset, i__1, i__2;
   doublereal d__1, d__2, d__3, d__4, d__5, d__6, d__7, d__8;
 
   /* Builtin functions */
@@ -2018,7 +2018,7 @@ doublereal *wc, *wrp;
   doublereal flag__, wbtw;
   integer j, k;
   doublereal delta, fleft, flmin, fcrit, frght, x0, x1, delta0,
-    delta1, wrtmp, deldif, ddelta;
+             delta1, wrtmp, deldif, ddelta;
 
 /*     we are solving for phase 0 here where indexes will likely be */
 /*       0 - water */
diff --git a/pfsimulator/parflow_lib/sadvection_godunov.c b/pfsimulator/parflow_lib/sadvection_godunov.c
index da3c5f933..7559f3e29 100644
--- a/pfsimulator/parflow_lib/sadvection_godunov.c
+++ b/pfsimulator/parflow_lib/sadvection_godunov.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
@@ -134,15 +134,15 @@ void     SatGodunov(
   SubregionArray   *subregion_array;
 
   Subgrid          *subgrid,
-    *well_subgrid,
-    *tmp_subgrid;
+                   *well_subgrid,
+                   *tmp_subgrid;
   Subregion        *subregion;
   Subvector        *subvector,
-    *subvector_scal,
-    *subvector_rhs,
-    *subvector_xvel,
-    *subvector_yvel,
-    *subvector_zvel;
+                   *subvector_scal,
+                   *subvector_rhs,
+                   *subvector_xvel,
+                   *subvector_yvel,
+                   *subvector_zvel;
 
   ComputePkg       *compute_pkg;
   Region           *compute_reg = NULL;
@@ -152,10 +152,10 @@ void     SatGodunov(
   int nx, ny, nz;
   double dx, dy, dz;
   int nx_s, ny_s, nz_s,
-    nx_w, ny_w, nz_w,
-    nx_xv, ny_xv, nz_xv,
-    nx_yv, ny_yv, nz_yv,
-    nx_zv, ny_zv, nz_zv;
+      nx_w, ny_w, nz_w,
+      nx_xv, ny_xv, nz_xv,
+      nx_yv, ny_yv, nz_yv,
+      nx_zv, ny_zv, nz_zv;
 
   int i, j, k, si, wi, xi, yi, zi;
   int index, flopest;
diff --git a/pfsimulator/parflow_lib/scale.c b/pfsimulator/parflow_lib/scale.c
index 371c1fa4a..ab6b6ff0d 100644
--- a/pfsimulator/parflow_lib/scale.c
+++ b/pfsimulator/parflow_lib/scale.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * The Matrix vector multiplication routine
diff --git a/pfsimulator/parflow_lib/select_time_step.c b/pfsimulator/parflow_lib/select_time_step.c
index fa1fb455b..b1929c0a0 100644
--- a/pfsimulator/parflow_lib/select_time_step.c
+++ b/pfsimulator/parflow_lib/select_time_step.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
diff --git a/pfsimulator/parflow_lib/set_problem_data.c b/pfsimulator/parflow_lib/set_problem_data.c
index 32d105ad8..7e2f7a08c 100644
--- a/pfsimulator/parflow_lib/set_problem_data.c
+++ b/pfsimulator/parflow_lib/set_problem_data.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
@@ -49,10 +49,13 @@ typedef struct {
   PFModule  *permeability;
   PFModule  *porosity;
   PFModule  *wells;
+  PFModule *reservoirs;
   PFModule  *bc_pressure;
   PFModule  *specific_storage;  //sk
   PFModule  *x_slope;  //sk
   PFModule  *y_slope;  //sk
+  PFModule  *wc_x;
+  PFModule  *wc_y;
   PFModule  *mann;  //sk
   PFModule  *dz_mult;   //RMM
   PFModule  *FBx;   //RMM
@@ -85,10 +88,13 @@ void          SetProblemData(
   PFModule      *permeability = (instance_xtra->permeability);
   PFModule      *porosity = (instance_xtra->porosity);
   PFModule      *wells = (instance_xtra->wells);
+  PFModule      *reservoirs = (instance_xtra->reservoirs);
   PFModule      *bc_pressure = (instance_xtra->bc_pressure);
   PFModule      *specific_storage = (instance_xtra->specific_storage);    //sk
   PFModule      *x_slope = (instance_xtra->x_slope);         //sk
   PFModule      *y_slope = (instance_xtra->y_slope);         //sk
+  PFModule      *wc_x = (instance_xtra->wc_x);
+  PFModule      *wc_y = (instance_xtra->wc_y);
   PFModule      *mann = (instance_xtra->mann);            //sk
   PFModule      *dz_mult = (instance_xtra->dz_mult);                //rmm
   PFModule      *FBx = (instance_xtra->FBx);                //rmm
@@ -98,7 +104,7 @@ void          SetProblemData(
   PFModule      *real_space_z = (instance_xtra->real_space_z);
 
   /* Note: the order in which these modules are called is important */
-  PFModuleInvokeType(WellPackageInvoke, wells, (problem_data));
+
   if ((instance_xtra->site_data_not_formed))
   {
     PFModuleInvokeType(GeometriesInvoke, geometries, (problem_data));
@@ -129,6 +135,14 @@ void          SetProblemData(
                        (problem_data,
                         ProblemDataTSlopeY(problem_data),
                         ProblemDataPorosity(problem_data)));
+    PFModuleInvokeType(ChannelWidthInvoke, wc_x,
+                       (problem_data,
+                        ProblemDataChannelWidthX(problem_data),
+                        ProblemDataPorosity(problem_data)));
+    PFModuleInvokeType(ChannelWidthInvoke, wc_y,
+                       (problem_data,
+                        ProblemDataChannelWidthY(problem_data),
+                        ProblemDataPorosity(problem_data)));
     PFModuleInvokeType(ManningsInvoke, mann,                   //sk
                        (problem_data,
                         ProblemDataMannings(problem_data),
@@ -155,7 +169,8 @@ void          SetProblemData(
 
     (instance_xtra->site_data_not_formed) = 0;
   }
-
+  PFModuleInvokeType(WellPackageInvoke, wells, (problem_data));
+  PFModuleInvokeType(ReservoirPackageInvoke, reservoirs, (problem_data));
   PFModuleInvokeType(BCPressurePackageInvoke, bc_pressure, (problem_data));
 }
 
@@ -229,6 +244,14 @@ PFModule  *SetProblemDataInitInstanceXtra(
     (instance_xtra->y_slope) =                                       //sk
                                PFModuleNewInstanceType(SlopeInitInstanceXtraInvoke,
                                                        ProblemYSlope(problem), (grid, grid2d));
+
+    (instance_xtra->wc_x) =
+      PFModuleNewInstanceType(ChannelWidthInitInstanceXtraInvoke,
+                              ProblemXChannelWidth(problem), (grid, grid2d));
+    (instance_xtra->wc_y) =
+      PFModuleNewInstanceType(ChannelWidthInitInstanceXtraInvoke,
+                              ProblemYChannelWidth(problem), (grid, grid2d));
+
     (instance_xtra->mann) =                                       //sk
                             PFModuleNewInstanceType(ManningsInitInstanceXtraInvoke,
                                                     ProblemMannings(problem), (grid, grid2d));
@@ -250,6 +273,9 @@ PFModule  *SetProblemDataInitInstanceXtra(
     (instance_xtra->wells) =
       PFModuleNewInstance(ProblemWellPackage(problem), ());
 
+    (instance_xtra->reservoirs) =
+      PFModuleNewInstance(ProblemReservoirPackage(problem), ());
+
     (instance_xtra->bc_pressure) =
       PFModuleNewInstanceType(BCPressurePackageInitInstanceXtraInvoke,
                               ProblemBCPressurePackage(problem), (problem));
@@ -271,6 +297,10 @@ PFModule  *SetProblemDataInitInstanceXtra(
                               (instance_xtra->x_slope), (grid, grid2d));        //sk
     PFModuleReNewInstanceType(SlopeInitInstanceXtraInvoke,
                               (instance_xtra->y_slope), (grid, grid2d));        //sk
+    PFModuleReNewInstanceType(ChannelWidthInitInstanceXtraInvoke,
+                              (instance_xtra->wc_x), (grid, grid2d));
+    PFModuleReNewInstanceType(ChannelWidthInitInstanceXtraInvoke,
+                              (instance_xtra->wc_y), (grid, grid2d));
     PFModuleReNewInstanceType(ManningsInitInstanceXtraInvoke,
                               (instance_xtra->mann), (grid, grid2d));        //sk
     PFModuleReNewInstance((instance_xtra->dz_mult), ());        //RMM
@@ -280,6 +310,7 @@ PFModule  *SetProblemDataInitInstanceXtra(
 
     PFModuleReNewInstance((instance_xtra->real_space_z), ());
     PFModuleReNewInstance((instance_xtra->wells), ());
+    PFModuleReNewInstance((instance_xtra->reservoirs), ());
     PFModuleReNewInstanceType(BCPressurePackageInitInstanceXtraInvoke,
                               (instance_xtra->bc_pressure), (problem));
   }
@@ -304,6 +335,7 @@ void  SetProblemDataFreeInstanceXtra()
   {
     PFModuleFreeInstance(instance_xtra->bc_pressure);
     PFModuleFreeInstance(instance_xtra->wells);
+    PFModuleFreeInstance(instance_xtra->reservoirs);
 
     PFModuleFreeInstance(instance_xtra->geometries);
     PFModuleFreeInstance(instance_xtra->domain);
@@ -312,6 +344,8 @@ void  SetProblemDataFreeInstanceXtra()
     PFModuleFreeInstance(instance_xtra->specific_storage);       //sk
     PFModuleFreeInstance(instance_xtra->x_slope);       //sk
     PFModuleFreeInstance(instance_xtra->y_slope);       //sk
+    PFModuleFreeInstance(instance_xtra->wc_x);
+    PFModuleFreeInstance(instance_xtra->wc_y);
     PFModuleFreeInstance(instance_xtra->mann);       //sk
     PFModuleFreeInstance(instance_xtra->dz_mult);       // RMM
     PFModuleFreeInstance(instance_xtra->FBx);       // RMM
diff --git a/pfsimulator/parflow_lib/sim_shear.c b/pfsimulator/parflow_lib/sim_shear.c
index 1d7197849..087bd5e60 100644
--- a/pfsimulator/parflow_lib/sim_shear.c
+++ b/pfsimulator/parflow_lib/sim_shear.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *****************************************************************************/
@@ -151,13 +151,13 @@ double         **SimShear(
 
         switch (type)
         {
-          case 1:
-            shear_array[ishear] = pfmin(z, shear_array[ishear]);
-            break;
+            case 1:
+              shear_array[ishear] = pfmin(z, shear_array[ishear]);
+              break;
 
-          case 2:
-            shear_array[ishear] = pfmax(z, shear_array[ishear]);
-            break;
+            case 2:
+              shear_array[ishear] = pfmax(z, shear_array[ishear]);
+              break;
         }
 
         shear_min[is] = pfmin(shear_min[is], shear_array[ishear]);
diff --git a/pfsimulator/parflow_lib/solver.c b/pfsimulator/parflow_lib/solver.c
index 601e40f0e..05b41312a 100644
--- a/pfsimulator/parflow_lib/solver.c
+++ b/pfsimulator/parflow_lib/solver.c
@@ -1,30 +1,30 @@
 /*
  * BHEADER**********************************************************************
  *
- * Copyright (c) 1995-2009, Lawrence Livermore National Security,
- * LLC. Produced at the Lawrence Livermore National Laboratory. Written
- * by the Parflow Team (see the CONTRIBUTORS file)
- * <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+ *  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+ *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+ *  by the Parflow Team (see the CONTRIBUTORS file)
+ *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
  *
- * This file is part of Parflow. For details, see
- * http://www.llnl.gov/casc/parflow
+ *  This file is part of Parflow. For details, see
+ *  http://www.llnl.gov/casc/parflow
  *
- * Please read the COPYRIGHT file or Our Notice and the LICENSE file
- * for the GNU Lesser General Public License.
+ *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+ *  for the GNU Lesser General Public License.
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License (as published
- * by the Free Software Foundation) version 2.1 dated February 1999.
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License (as published
+ *  by the Free Software Foundation) version 2.1 dated February 1999.
  *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- * and conditions of the GNU General Public License for more details.
+ *  This program is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+ *  and conditions of the GNU General Public License for more details.
  *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- * USA
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ *  USA
  **********************************************************************EHEADER
  * */
 /*****************************************************************************
@@ -124,25 +124,25 @@ NewSolver()
     {
       case 0:
       {
-	amps_ThreadLocal(Solver_module) = PFModuleNewModuleType(SolverNewPublicXtraInvoke, SolverRichards, ("Solver"));
-	break;
+        amps_ThreadLocal(Solver_module) = PFModuleNewModuleType(SolverNewPublicXtraInvoke, SolverRichards, ("Solver"));
+        break;
       }
-      
+
       case 1:
       {
-	amps_ThreadLocal(Solver_module) = PFModuleNewModuleType(SolverNewPublicXtraInvoke, SolverDiffusion, ("Solver"));
-	break;
+        amps_ThreadLocal(Solver_module) = PFModuleNewModuleType(SolverNewPublicXtraInvoke, SolverDiffusion, ("Solver"));
+        break;
       }
-      
+
       case 2:
       {
-	amps_ThreadLocal(Solver_module) = PFModuleNewModuleType(SolverNewPublicXtraInvoke, SolverImpes, ("Solver"));
-	break;
+        amps_ThreadLocal(Solver_module) = PFModuleNewModuleType(SolverNewPublicXtraInvoke, SolverImpes, ("Solver"));
+        break;
       }
-      
+
       default:
       {
-	InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+        InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
       }
     }
   }
diff --git a/pfsimulator/parflow_lib/solver.h b/pfsimulator/parflow_lib/solver.h
index df3f7fc27..d038454d9 100644
--- a/pfsimulator/parflow_lib/solver.h
+++ b/pfsimulator/parflow_lib/solver.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Stuctures for the solver.
diff --git a/pfsimulator/parflow_lib/solver_impes.c b/pfsimulator/parflow_lib/solver_impes.c
index e8ebe2d8d..b5e0f53b8 100644
--- a/pfsimulator/parflow_lib/solver_impes.c
+++ b/pfsimulator/parflow_lib/solver_impes.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /***************************************************************************
 *
@@ -283,7 +283,7 @@ void      SolverImpes()
 
   if (public_xtra->write_silo_subsurf_data)
   {
-    strcpy(file_postfix,"");
+    strcpy(file_postfix, "");
     sprintf(file_type, "perm_x");
     WriteSilo(file_prefix, file_type, file_postfix, ProblemDataPermeabilityX(problem_data),
               t, 0, "PermeabilityX");
@@ -1150,28 +1150,28 @@ void      SolverImpes()
           }
 
           /* put call to CRUNCHFLOW here @RMM */
-	  /* int is; */
-	  /* ForSubgridI(is, GridSubgrids(grid)) */
-	  /* { */
-	  /*   double dx, dy, dz; */
-	  /*   int nx, ny, nz, nx_f, ny_f, nz_f, nz_rz, ip, ix, iy, iz; */
-	  /*   int x, y, z; */
-	  
-	  /*   // @RMM - dummy variables for calling CRUNCHFLOW */
-	  /*   nx = SubgridNX(subgrid); */
-	  /*   ny = SubgridNY(subgrid); */
-	  /*   nz = SubgridNZ(subgrid); */
-	  
-	  /*   ix = SubgridIX(subgrid); */
-	  /*   iy = SubgridIY(subgrid); */
-	  /*   iz = SubgridIZ(subgrid); */
-	  
-	  /*   dx = SubgridDX(subgrid); */
-	  /*   dy = SubgridDY(subgrid); */
-	  /*   dz = SubgridDZ(subgrid); */
-	  
-	  /*   CALL_CRUNCHFLOW(); */
-	  /* } */
+          /* int is; */
+          /* ForSubgridI(is, GridSubgrids(grid)) */
+          /* { */
+          /*   double dx, dy, dz; */
+          /*   int nx, ny, nz, nx_f, ny_f, nz_f, nz_rz, ip, ix, iy, iz; */
+          /*   int x, y, z; */
+
+          /*   // @RMM - dummy variables for calling CRUNCHFLOW */
+          /*   nx = SubgridNX(subgrid); */
+          /*   ny = SubgridNY(subgrid); */
+          /*   nz = SubgridNZ(subgrid); */
+
+          /*   ix = SubgridIX(subgrid); */
+          /*   iy = SubgridIY(subgrid); */
+          /*   iz = SubgridIZ(subgrid); */
+
+          /*   dx = SubgridDX(subgrid); */
+          /*   dy = SubgridDY(subgrid); */
+          /*   dz = SubgridDZ(subgrid); */
+
+          /*   CALL_CRUNCHFLOW(); */
+          /* } */
         }
 
         /* Print the concentration values at this time-step? */
@@ -1264,7 +1264,7 @@ void      SolverImpes()
 
       if (public_xtra->write_silo_press)
       {
-	strcpy(file_postfix, "");
+        strcpy(file_postfix, "");
         sprintf(file_type, "press");
         WriteSilo(file_prefix, file_type, file_postfix, pressure,
                   t, file_number, "Pressure");
@@ -1503,7 +1503,7 @@ PFModule *SolverImpesInitInstanceXtra()
   // SGS TODO total_mobility_sz is not being set anywhere so initialized to 0 here.
   int total_mobility_sz = 0;
   int pressure_sz, velocity_sz, satur_sz = 0,
-    concen_sz, temp_data_size, sz;
+      concen_sz, temp_data_size, sz;
   int is_multiphase;
 
   int i;
@@ -1823,7 +1823,7 @@ PFModule *SolverImpesInitInstanceXtra()
   int size_advect = PFModuleSizeOfTempData(instance_xtra->advect_concen);
   temp_data_placeholder += pfmax(
                                  size_retardation,
-				 size_advect
+                                 size_advect
                                  );
   /* set temporary vector data used for advection */
 
diff --git a/pfsimulator/parflow_lib/solver_lb.c b/pfsimulator/parflow_lib/solver_lb.c
index 67494c80e..1d7be4b22 100644
--- a/pfsimulator/parflow_lib/solver_lb.c
+++ b/pfsimulator/parflow_lib/solver_lb.c
@@ -1,11 +1,30 @@
-/*BHEADER*********************************************************************
-* (c) 1996   The Regents of the University of California
+/*BHEADER**********************************************************************
 *
-* See the file COPYRIGHT_and_DISCLAIMER for a complete copyright
-* notice, contact person, and disclaimer.
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
 *
-* $Revision: 1.1.1.1 $
-*********************************************************************EHEADER*/
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /****************************************************************************
  *
@@ -1328,7 +1347,7 @@ PFModule *SolverDiffusionInitInstanceXtra()
 
   double       *temp_data, *temp_data_placeholder;
   int total_mobility_sz, pressure_sz, velocity_sz, satur_sz = 0,
-    concen_sz, temp_data_size, sz;
+      concen_sz, temp_data_size, sz;
   int is_multiphase;
 
   int i;
@@ -1615,9 +1634,9 @@ PFModule *SolverDiffusionInitInstanceXtra()
                             (NULL, NULL, temp_data_placeholder));
   int size_retardation = PFModuleSizeOfTempData(instance_xtra->retardation);
   int size_advect = PFModuleSizeOfTempData(instance_xtra->advect_concen);
-  
+
   temp_data_placeholder += pfmax(size_retardation,
-				 size_advect
+                                 size_advect
                                  );
 
   temp_data += temp_data_size;
diff --git a/pfsimulator/parflow_lib/solver_richards.c b/pfsimulator/parflow_lib/solver_richards.c
index f5e2e8e0c..eb001f0d8 100644
--- a/pfsimulator/parflow_lib/solver_richards.c
+++ b/pfsimulator/parflow_lib/solver_richards.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /****************************************************************************
  *
@@ -73,9 +73,11 @@ typedef struct {
   int terrain_following_grid;   /* @RMM flag for terrain following grid in NL fn eval, sets sslopes=toposl */
   int variable_dz;              /* @RMM flag for variable dz-multipliers */
 
+  int print_initial_conditions; /* print the initial conditions, turning this off is useful during restart */
   int print_subsurf_data;       /* print permeability/porosity? */
   int print_press;              /* print pressures? */
   int print_slopes;             /* print slopes? */
+  int print_channelwidth;       /* print channel width? */
   int print_mannings;           /* print mannings? */
   int print_specific_storage;   /* print spec storage? */
   int print_top;                /* print top? */
@@ -84,6 +86,7 @@ typedef struct {
   int print_mask;               /* print mask? */
   int print_concen;             /* print concentrations? */
   int print_wells;              /* print well data? */
+  int print_reservoirs;         /* print reservoir data? */
   int print_dzmult;             /* print dz multiplier? */
   int print_evaptrans;          /* print evaptrans? */
   int print_evaptrans_sum;      /* print evaptrans_sum? */
@@ -150,7 +153,7 @@ typedef struct {
   char *clm_metfile;            /* File name for 1D forcing *or* base name for 2D forcing */
   char *clm_metpath;            /* Path to CLM met forcing file(s) */
   double *sw1d, *lw1d, *prcp1d, /* 1D forcing variables */
-    *tas1d, *u1d, *v1d, *patm1d, *qatm1d, *lai1d, *sai1d, *z0m1d, *displa1d;    /* BH: added lai, sai, z0m, displa */
+         *tas1d, *u1d, *v1d, *patm1d, *qatm1d, *lai1d, *sai1d, *z0m1d, *displa1d; /* BH: added lai, sai, z0m, displa */
 
   int clm_beta_function;        /* CLM evap function for var sat 0=none, 1=linear, 2=cos */
   double clm_res_sat;           /* CLM residual saturation in soil sat units [-] */
@@ -353,9 +356,12 @@ SetupRichards(PFModule * this_module)
   PFModule *phase_density = (instance_xtra->phase_density);
   PFModule *problem_saturation = (instance_xtra->problem_saturation);
 
+  int print_initial_conditions = (public_xtra->print_initial_conditions); // Print the initial conditions
+
   int print_press = (public_xtra->print_press);
   int print_satur = (public_xtra->print_satur);
   int print_wells = (public_xtra->print_wells);
+  int print_reservoirs = (public_xtra->print_reservoirs);
   int print_velocities = (public_xtra->print_velocities);       //jjb
 
   ProblemData *problem_data = (instance_xtra->problem_data);
@@ -385,9 +391,10 @@ SetupRichards(PFModule * this_module)
 
   int any_file_dumped;
 
+  char filename[128];
+
 #ifdef HAVE_CLM
   /* IMF: for CLM met forcings (local to SetupRichards) */
-  char filename[128];
   int n, nc, c;                 /*Added c BH */
   int ch;
   double sw, lw, prcp, tas, u, v, patm, qatm, lai, sai, z0m, displa;    // forcing vars added vegetation BH
@@ -416,7 +423,6 @@ SetupRichards(PFModule * this_module)
           !clmForcingFields[ff].vegetative ||
           (public_xtra->clm_metforce == 3 && public_xtra->clm_forc_veg == 1))
       {
-
         MetadataAddForcingField(
                                 js_inputs,
                                 clmForcingFields[ff].field_name,
@@ -456,7 +462,7 @@ SetupRichards(PFModule * this_module)
   PFModuleInvokeType(SetProblemDataInvoke, set_problem_data, (problem_data));
   ComputeTop(problem, problem_data);
 
-  if(public_xtra->print_top || public_xtra->write_silo_top)
+  if (public_xtra->print_top || public_xtra->write_silo_top)
   {
     ComputePatchTop(problem, problem_data);
   }
@@ -506,7 +512,7 @@ SetupRichards(PFModule * this_module)
                   ProblemDataSpecificStorage(problem_data));
 
     PFModuleOutputStaticType(SaturationOutputStaticInvoke, ProblemSaturation(problem), (file_prefix, problem_data));
-    
+
     // Now add metadata entries:
     static const char* permeability_filenames[] = {
       "perm_x", "perm_y", "perm_z"
@@ -610,6 +616,26 @@ SetupRichards(PFModule * this_module)
                            slope_filenames);
   }
 
+  if (public_xtra->print_channelwidth)
+  {
+    strcpy(file_postfix, "wc_x");
+    WritePFBinary(file_prefix, file_postfix,
+                  ProblemDataChannelWidthX(problem_data));
+
+    strcpy(file_postfix, "wc_y");
+    WritePFBinary(file_prefix, file_postfix,
+                  ProblemDataChannelWidthY(problem_data));
+
+    static const char* filenames[] = {
+      "wc_x", "wc_y"
+    };
+    MetadataAddStaticField(
+                           js_inputs, file_prefix, "channel_width", NULL, "cell", "surface",
+                           sizeof(filenames) / sizeof(filenames[0]),
+                           filenames);
+  }
+
+
   if (public_xtra->write_silo_slopes)
   {
     strcpy(file_postfix, "");
@@ -776,6 +802,7 @@ SetupRichards(PFModule * this_module)
     print_press = 0;
     print_satur = 0;
     print_wells = 0;
+    print_reservoirs = 0;
     print_velocities = 0;       //jjb
   }
 
@@ -851,13 +878,27 @@ SetupRichards(PFModule * this_module)
 
     if (public_xtra->evap_trans_file)
     {
-      //sprintf(filename, "%s", public_xtra->evap_trans_filename);
-      //printf("%s %s \n",filename, public_xtra -> evap_trans_filename);
+      sprintf(filename, "%s", public_xtra->evap_trans_filename);
       ReadPFBinary(public_xtra->evap_trans_filename, instance_xtra->evap_trans);
 
       handle = InitVectorUpdate(instance_xtra->evap_trans, VectorUpdateAll);
       FinalizeVectorUpdate(handle);
     }
+#ifndef HAVE_CLM
+    //printf("DEBUG: public_xtra->nc_evap_trans_file_transient = %i \n", public_xtra->nc_evap_trans_file_transient);
+    if (public_xtra->nc_evap_trans_file_transient)
+    {
+      strcpy(filename, public_xtra->nc_evap_trans_filename);
+      /*KKu: evaptrans is the name of the variable expected in NetCDF file */
+      /*Here looping similar to pfb is not implemented. All steps are assumed to be
+       * present in the single NetCDF file*/
+      //printf("DEBUG: read evap_trans file %s for step %i \n", filename, instance_xtra->file_number);
+      ReadPFNC(filename, instance_xtra->evap_trans, "evaptrans", instance_xtra->file_number, 3);
+      //printf("DEBUG: evap_trans max min: %e %e \n",PFVMax(instance_xtra->evap_trans),PFVMin(instance_xtra->evap_trans));
+      handle = InitVectorUpdate(instance_xtra->evap_trans, VectorUpdateAll);
+      FinalizeVectorUpdate(handle);
+    }
+#endif
 
     /* IMF: the following are only used w/ CLM */
 #ifdef HAVE_CLM
@@ -1040,7 +1081,6 @@ SetupRichards(PFModule * this_module)
       if (public_xtra->clm_forc_veg == 1)
       {
         /*Reading file LAI */ /*BH*/
-        /*sprintf(filename, "%s/%s", public_xtra -> clm_metpath, public_xtra -> clm_metfile); */
         sprintf(filename, "%s/%s", public_xtra->clm_metpath,
                 "lai.dat");
 
@@ -1052,7 +1092,6 @@ SetupRichards(PFModule * this_module)
         }
         /*assume nc remains the same BH */
         // Read 1D met file to arrays of length nc
-        //(public_xtra -> lai1d) = ctalloc(double,nc*18);
         if ((metf1d = amps_SFopen(filename, "r")) == NULL)
         {
           amps_Printf("Error: can't open file %s \n", filename);
@@ -1072,7 +1111,6 @@ SetupRichards(PFModule * this_module)
         amps_SFclose(metf1d);
 
         /*Reading file SAI */ /*BH*/
-        /*sprintf(filename, "%s/%s", public_xtra -> clm_metpath, public_xtra -> clm_metfile); */
         sprintf(filename, "%s/%s", public_xtra->clm_metpath,
                 "sai.dat");
 
@@ -1084,7 +1122,6 @@ SetupRichards(PFModule * this_module)
         }
         /*assume nc remains the same BH */
         // Read 1D met file to arrays of length nc
-        //(public_xtra -> sai1d) = ctalloc(double,nc*18);
         if ((metf1d = amps_SFopen(filename, "r")) == NULL)
         {
           amps_Printf("Error: can't open file %s \n", filename);
@@ -1116,7 +1153,6 @@ SetupRichards(PFModule * this_module)
         }
         /*assume nc remains the same BH */
         // Read 1D met file to arrays of length nc
-        //(public_xtra -> z0m1d) = ctalloc(double,nc*18);
         if ((metf1d = amps_SFopen(filename, "r")) == NULL)
         {
           amps_Printf("Error: can't open file %s \n", filename);
@@ -1136,7 +1172,6 @@ SetupRichards(PFModule * this_module)
         amps_SFclose(metf1d);
 
         /*Reading file displa */ /*BH*/
-        /*sprintf(filename, "%s/%s", public_xtra -> clm_metpath, public_xtra -> clm_metfile); */
         sprintf(filename, "%s/%s", public_xtra->clm_metpath,
                 "displa.dat");
 
@@ -1148,7 +1183,6 @@ SetupRichards(PFModule * this_module)
         }
         /*assume nc remains the same BH */
         // Read 1D met file to arrays of length nc
-        //(public_xtra -> displa1d) = ctalloc(double,nc*18);
         if ((metf1d = amps_SFopen(filename, "r")) == NULL)
         {
           amps_Printf("Error: can't open file %s \n", filename);
@@ -1211,282 +1245,380 @@ SetupRichards(PFModule * this_module)
 
     any_file_dumped = 0;
 
-    /*-------------------------------------------------------------------
-     * Print out the initial well data?
-     *-------------------------------------------------------------------*/
-
-    if (print_wells)
-    {
-      WriteWells(file_prefix,
-                 problem,
-                 ProblemDataWellData(problem_data),
-                 t, WELLDATA_WRITEHEADER);
-    }
-    sprintf(nc_postfix, "%05d", instance_xtra->file_number);
-    if (public_xtra->write_netcdf_press || public_xtra->write_netcdf_satur
-        || public_xtra->write_netcdf_mannings
-        || public_xtra->write_netcdf_subsurface
-        || public_xtra->write_netcdf_slopes
-        || public_xtra->write_netcdf_mask
-        || public_xtra->write_netcdf_dzmult)
+    if (print_initial_conditions)
     {
-      WritePFNC(file_prefix, nc_postfix, t, instance_xtra->pressure,
-                public_xtra->numVarTimeVariant, "time", 1, true,
-                public_xtra->numVarIni);
-    }
-    /*-----------------------------------------------------------------
-     * Print out the initial pressures?
-     *-----------------------------------------------------------------*/
+      /*-------------------------------------------------------------------
+       * Print out the initial reservoir data?
+       *-------------------------------------------------------------------*/
+      if (print_reservoirs)
+      {
+        WriteReservoirs("ReservoirsOutput",
+                        problem,
+                        ProblemDataReservoirData(problem_data),
+                        t, RESERVOIRDATA_WRITEHEADER);
+      }
+      /*-------------------------------------------------------------------
+       * Print out the initial well data?
+       *-------------------------------------------------------------------*/
+      if (print_wells)
+      {
+        WriteWells(file_prefix,
+                   problem,
+                   ProblemDataWellData(problem_data),
+                   t, WELLDATA_WRITEHEADER);
+      }
+      sprintf(nc_postfix, "%05d", instance_xtra->file_number);
+      if (public_xtra->write_netcdf_press || public_xtra->write_netcdf_satur
+          || public_xtra->write_netcdf_mannings
+          || public_xtra->write_netcdf_subsurface
+          || public_xtra->write_netcdf_slopes
+          || public_xtra->write_netcdf_mask
+          || public_xtra->write_netcdf_dzmult)
+      {
+        WritePFNC(file_prefix, nc_postfix, t, instance_xtra->pressure,
+                  public_xtra->numVarTimeVariant, "time", 1, true,
+                  public_xtra->numVarIni);
+      }
+      /*-----------------------------------------------------------------
+       * Print out the initial pressures?
+       *-----------------------------------------------------------------*/
 
-    if (print_press)
-    {
-      sprintf(file_postfix, "press.%05d", instance_xtra->file_number);
-      WritePFBinary(file_prefix, file_postfix, instance_xtra->pressure);
-      any_file_dumped = 1;
+      if (print_press)
+      {
+        sprintf(file_postfix, "press.%05d", instance_xtra->file_number);
+        WritePFBinary(file_prefix, file_postfix, instance_xtra->pressure);
+        any_file_dumped = 1;
 
-      static const char* press_filenames[] = {
-        "press"
-      };
-      MetadataAddDynamicField(
-                              js_outputs, file_prefix, t, 0, "pressure", "m", "cell", "subsurface",
-                              sizeof(press_filenames) / sizeof(press_filenames[0]),
-                              press_filenames);
-    }
+        static const char* press_filenames[] = {
+          "press"
+        };
+        MetadataAddDynamicField(
+                                js_outputs, file_prefix, t, 0, "pressure", "m", "cell", "subsurface",
+                                sizeof(press_filenames) / sizeof(press_filenames[0]),
+                                press_filenames);
+      }
 
-    if (public_xtra->write_silo_press)
-    {
-      sprintf(file_postfix, "%05d", instance_xtra->file_number);
-      strcpy(file_type, "press");
-      WriteSilo(file_prefix, file_type, file_postfix,
-                instance_xtra->pressure, t, instance_xtra->file_number,
-                "Pressure");
-      any_file_dumped = 1;
-    }
+      if (public_xtra->write_silo_press)
+      {
+        sprintf(file_postfix, "%05d", instance_xtra->file_number);
+        strcpy(file_type, "press");
+        WriteSilo(file_prefix, file_type, file_postfix,
+                  instance_xtra->pressure, t, instance_xtra->file_number,
+                  "Pressure");
+        any_file_dumped = 1;
+      }
 
-    if (public_xtra->write_silopmpio_press)
-    {
-      sprintf(file_postfix, "%05d", instance_xtra->file_number);
-      strcpy(file_type, "press");
-      WriteSiloPMPIO(file_prefix, file_type, file_postfix,
-                     instance_xtra->pressure, t,
-                     instance_xtra->file_number, "Pressure");
-      any_file_dumped = 1;
-    }
-    if (public_xtra->write_netcdf_press)
-    {
-      sprintf(file_postfix, "press.%05d", instance_xtra->file_number);
-      sprintf(nc_postfix, "%05d", instance_xtra->file_number);
-      WritePFNC(file_prefix, nc_postfix, t, instance_xtra->pressure,
-                public_xtra->numVarTimeVariant, "pressure", 3, true,
-                public_xtra->numVarIni);
-      any_file_dumped = 1;
-    }
-    /*-----------------------------------------------------------------
-     * Print out the initial saturations?
-     *-----------------------------------------------------------------*/
+      if (public_xtra->write_silopmpio_press)
+      {
+        sprintf(file_postfix, "%05d", instance_xtra->file_number);
+        strcpy(file_type, "press");
+        WriteSiloPMPIO(file_prefix, file_type, file_postfix,
+                       instance_xtra->pressure, t,
+                       instance_xtra->file_number, "Pressure");
+        any_file_dumped = 1;
+      }
+      if (public_xtra->write_netcdf_press)
+      {
+        sprintf(file_postfix, "press.%05d", instance_xtra->file_number);
+        sprintf(nc_postfix, "%05d", instance_xtra->file_number);
+        WritePFNC(file_prefix, nc_postfix, t, instance_xtra->pressure,
+                  public_xtra->numVarTimeVariant, "pressure", 3, true,
+                  public_xtra->numVarIni);
+        any_file_dumped = 1;
+      }
+      /*-----------------------------------------------------------------
+       * Print out the initial saturations?
+       *-----------------------------------------------------------------*/
 
-    if (print_satur)
-    {
-      sprintf(file_postfix, "satur.%05d", instance_xtra->file_number);
-      WritePFBinary(file_prefix, file_postfix,
-                    instance_xtra->saturation);
-      any_file_dumped = 1;
+      if (print_satur)
+      {
+        sprintf(file_postfix, "satur.%05d", instance_xtra->file_number);
+        WritePFBinary(file_prefix, file_postfix,
+                      instance_xtra->saturation);
+        any_file_dumped = 1;
 
-      static const char* satur_filenames[] = {
-        "satur"
-      };
-      MetadataAddDynamicField(
-                              js_outputs, file_prefix, t, 0, "saturation", NULL, "cell", "subsurface",
-                              sizeof(satur_filenames) / sizeof(satur_filenames[0]),
-                              satur_filenames);
-    }
+        static const char* satur_filenames[] = {
+          "satur"
+        };
+        MetadataAddDynamicField(
+                                js_outputs, file_prefix, t, 0, "saturation", NULL, "cell", "subsurface",
+                                sizeof(satur_filenames) / sizeof(satur_filenames[0]),
+                                satur_filenames);
+      }
 
-    if (public_xtra->write_silo_satur)
-    {
-      sprintf(file_postfix, "%05d", instance_xtra->file_number);
-      strcpy(file_type, "satur");
-      WriteSilo(file_prefix, file_type, file_postfix,
-                instance_xtra->saturation, t, instance_xtra->file_number,
-                "Saturation");
-      any_file_dumped = 1;
-    }
+      if (public_xtra->write_silo_satur)
+      {
+        sprintf(file_postfix, "%05d", instance_xtra->file_number);
+        strcpy(file_type, "satur");
+        WriteSilo(file_prefix, file_type, file_postfix,
+                  instance_xtra->saturation, t, instance_xtra->file_number,
+                  "Saturation");
+        any_file_dumped = 1;
+      }
 
-    if (public_xtra->write_silopmpio_satur)
-    {
-      sprintf(file_postfix, "%05d", instance_xtra->file_number);
-      strcpy(file_type, "satur");
-      WriteSiloPMPIO(file_prefix, file_type, file_postfix,
-                     instance_xtra->saturation, t,
-                     instance_xtra->file_number, "Saturation");
-      any_file_dumped = 1;
-    }
-    if (public_xtra->write_netcdf_satur)
-    {
-      sprintf(file_postfix, "satur.%05d", instance_xtra->file_number);
-      sprintf(nc_postfix, "%05d", instance_xtra->file_number);
-      WritePFNC(file_prefix, nc_postfix, t, instance_xtra->saturation,
-                public_xtra->numVarTimeVariant, "saturation", 3, true,
-                public_xtra->numVarIni);
-      any_file_dumped = 1;
-    }
+      if (public_xtra->write_silopmpio_satur)
+      {
+        sprintf(file_postfix, "%05d", instance_xtra->file_number);
+        strcpy(file_type, "satur");
+        WriteSiloPMPIO(file_prefix, file_type, file_postfix,
+                       instance_xtra->saturation, t,
+                       instance_xtra->file_number, "Saturation");
+        any_file_dumped = 1;
+      }
+      if (public_xtra->write_netcdf_satur)
+      {
+        sprintf(file_postfix, "satur.%05d", instance_xtra->file_number);
+        sprintf(nc_postfix, "%05d", instance_xtra->file_number);
+        WritePFNC(file_prefix, nc_postfix, t, instance_xtra->saturation,
+                  public_xtra->numVarTimeVariant, "saturation", 3, true,
+                  public_xtra->numVarIni);
+        any_file_dumped = 1;
+      }
 
-    /*-----------------------------------------------------------------
-     * Print out Mannings in NetCDF?
-     *-----------------------------------------------------------------*/
-    if (public_xtra->write_netcdf_mannings)
-    {
-      sprintf(nc_postfix, "%05d", instance_xtra->file_number);
-      WritePFNC(file_prefix, nc_postfix, t,
-                ProblemDataMannings(problem_data),
-                public_xtra->numVarTimeVariant, "mannings", 2, true,
-                public_xtra->numVarIni);
-      any_file_dumped = 1;
-    }
+      /*-----------------------------------------------------------------
+       * Print out Mannings in NetCDF?
+       *-----------------------------------------------------------------*/
+      if (public_xtra->write_netcdf_mannings)
+      {
+        sprintf(nc_postfix, "%05d", instance_xtra->file_number);
+        WritePFNC(file_prefix, nc_postfix, t,
+                  ProblemDataMannings(problem_data),
+                  public_xtra->numVarTimeVariant, "mannings", 2, true,
+                  public_xtra->numVarIni);
+        any_file_dumped = 1;
+      }
 
-    /*-----------------------------------------------------------------
-     * Print out Subsurface data in NetCDF?
-     *-----------------------------------------------------------------*/
-    if (public_xtra->write_netcdf_subsurface)
-    {
-      sprintf(nc_postfix, "%05d", instance_xtra->file_number);
-      WritePFNC(file_prefix, nc_postfix, t,
-                ProblemDataPermeabilityX(problem_data),
-                public_xtra->numVarTimeVariant, "perm_x", 3, true,
-                public_xtra->numVarIni);
-      WritePFNC(file_prefix, nc_postfix, t,
-                ProblemDataPermeabilityY(problem_data),
-                public_xtra->numVarTimeVariant, "perm_y", 3, true,
-                public_xtra->numVarIni);
-      WritePFNC(file_prefix, nc_postfix, t,
-                ProblemDataPermeabilityZ(problem_data),
-                public_xtra->numVarTimeVariant, "perm_z", 3, true,
-                public_xtra->numVarIni);
-      WritePFNC(file_prefix, nc_postfix, t,
-                ProblemDataPorosity(problem_data),
-                public_xtra->numVarTimeVariant, "porosity", 3, true,
-                public_xtra->numVarIni);
-      WritePFNC(file_prefix, nc_postfix, t,
-                ProblemDataSpecificStorage(problem_data),
-                public_xtra->numVarTimeVariant, "specific_storage", 3,
-                true, public_xtra->numVarIni);
-      any_file_dumped = 1;
-    }
+      /*-----------------------------------------------------------------
+       * Print out Subsurface data in NetCDF?
+       *-----------------------------------------------------------------*/
+      if (public_xtra->write_netcdf_subsurface)
+      {
+        sprintf(nc_postfix, "%05d", instance_xtra->file_number);
+        WritePFNC(file_prefix, nc_postfix, t,
+                  ProblemDataPermeabilityX(problem_data),
+                  public_xtra->numVarTimeVariant, "perm_x", 3, true,
+                  public_xtra->numVarIni);
+        WritePFNC(file_prefix, nc_postfix, t,
+                  ProblemDataPermeabilityY(problem_data),
+                  public_xtra->numVarTimeVariant, "perm_y", 3, true,
+                  public_xtra->numVarIni);
+        WritePFNC(file_prefix, nc_postfix, t,
+                  ProblemDataPermeabilityZ(problem_data),
+                  public_xtra->numVarTimeVariant, "perm_z", 3, true,
+                  public_xtra->numVarIni);
+        WritePFNC(file_prefix, nc_postfix, t,
+                  ProblemDataPorosity(problem_data),
+                  public_xtra->numVarTimeVariant, "porosity", 3, true,
+                  public_xtra->numVarIni);
+        WritePFNC(file_prefix, nc_postfix, t,
+                  ProblemDataSpecificStorage(problem_data),
+                  public_xtra->numVarTimeVariant, "specific_storage", 3,
+                  true, public_xtra->numVarIni);
+        any_file_dumped = 1;
+      }
 
-    /*-----------------------------------------------------------------
-     * Print out Slopes in NetCDF?
-     *-----------------------------------------------------------------*/
-    if (public_xtra->write_netcdf_slopes)
-    {
-      sprintf(nc_postfix, "%05d", instance_xtra->file_number);
-      WritePFNC(file_prefix, nc_postfix, t,
-                ProblemDataTSlopeX(problem_data),
-                public_xtra->numVarTimeVariant, "slopex", 2, true,
-                public_xtra->numVarIni);
-      WritePFNC(file_prefix, nc_postfix, t,
-                ProblemDataTSlopeY(problem_data),
-                public_xtra->numVarTimeVariant, "slopey", 2, true,
-                public_xtra->numVarIni);
-      any_file_dumped = 1;
-    }
+      /*-----------------------------------------------------------------
+       * Print out Slopes in NetCDF?
+       *-----------------------------------------------------------------*/
+      if (public_xtra->write_netcdf_slopes)
+      {
+        sprintf(nc_postfix, "%05d", instance_xtra->file_number);
+        WritePFNC(file_prefix, nc_postfix, t,
+                  ProblemDataTSlopeX(problem_data),
+                  public_xtra->numVarTimeVariant, "slopex", 2, true,
+                  public_xtra->numVarIni);
+        WritePFNC(file_prefix, nc_postfix, t,
+                  ProblemDataTSlopeY(problem_data),
+                  public_xtra->numVarTimeVariant, "slopey", 2, true,
+                  public_xtra->numVarIni);
+        any_file_dumped = 1;
+      }
 
-    /*-----------------------------------------------------------------
-     * Print out dz multipliers in NetCDF?
-     *-----------------------------------------------------------------*/
-    if (public_xtra->write_netcdf_dzmult)
-    {
-      sprintf(nc_postfix, "%05d", instance_xtra->file_number);
-      WritePFNC(file_prefix, nc_postfix, t, instance_xtra->dz_mult,
-                public_xtra->numVarTimeVariant, "DZ_Multiplier", 3, true,
-                public_xtra->numVarIni);
-      any_file_dumped = 1;
-    }
+      /*-----------------------------------------------------------------
+       * Print out dz multipliers in NetCDF?
+       *-----------------------------------------------------------------*/
+      if (public_xtra->write_netcdf_dzmult)
+      {
+        sprintf(nc_postfix, "%05d", instance_xtra->file_number);
+        WritePFNC(file_prefix, nc_postfix, t, instance_xtra->dz_mult,
+                  public_xtra->numVarTimeVariant, "DZ_Multiplier", 3, true,
+                  public_xtra->numVarIni);
+        any_file_dumped = 1;
+      }
 
-    /*-----------------------------------------------------------------
-     * Print out mask?
-     *-----------------------------------------------------------------*/
+      /*-----------------------------------------------------------------
+       * Print out mask?
+       *-----------------------------------------------------------------*/
 
-    if (public_xtra->print_mask)
-    {
-      strcpy(file_postfix, "mask");
-      WritePFBinary(file_prefix, file_postfix, instance_xtra->mask);
-      any_file_dumped = 1;
+      if (public_xtra->print_mask)
+      {
+        strcpy(file_postfix, "mask");
+        WritePFBinary(file_prefix, file_postfix, instance_xtra->mask);
+        any_file_dumped = 1;
 
-      static const char* mask_filenames[] = {
-        "mask"
-      };
-      MetadataAddStaticField(
-                             js_inputs, file_prefix, "mask", NULL, "cell", "subsurface",
-                             sizeof(mask_filenames) / sizeof(mask_filenames[0]),
-                             mask_filenames);
-    }
+        static const char* mask_filenames[] = {
+          "mask"
+        };
+        MetadataAddStaticField(
+                               js_inputs, file_prefix, "mask", NULL, "cell", "subsurface",
+                               sizeof(mask_filenames) / sizeof(mask_filenames[0]),
+                               mask_filenames);
+      }
 
-    if (public_xtra->write_netcdf_mask)
-    {
-      sprintf(nc_postfix, "%05d", instance_xtra->file_number);
-      WritePFNC(file_prefix, nc_postfix, t, instance_xtra->mask,
-                public_xtra->numVarTimeVariant, "mask", 3, true,
-                public_xtra->numVarIni);
-      any_file_dumped = 1;
-    }
+      if (public_xtra->write_netcdf_mask)
+      {
+        sprintf(nc_postfix, "%05d", instance_xtra->file_number);
+        WritePFNC(file_prefix, nc_postfix, t, instance_xtra->mask,
+                  public_xtra->numVarTimeVariant, "mask", 3, true,
+                  public_xtra->numVarIni);
+        any_file_dumped = 1;
+      }
 
 
-    if (public_xtra->write_silo_mask)
-    {
-      strcpy(file_postfix, "");
-      strcpy(file_type, "mask");
-      WriteSilo(file_prefix, file_type, file_postfix,
-                instance_xtra->mask, t, instance_xtra->file_number,
-                "Mask");
-      any_file_dumped = 1;
-    }
+      if (public_xtra->write_silo_mask)
+      {
+        strcpy(file_postfix, "");
+        strcpy(file_type, "mask");
+        WriteSilo(file_prefix, file_type, file_postfix,
+                  instance_xtra->mask, t, instance_xtra->file_number,
+                  "Mask");
+        any_file_dumped = 1;
+      }
 
-    if (public_xtra->write_silopmpio_mask)
-    {
-      strcpy(file_postfix, "");
-      strcpy(file_type, "mask");
-      WriteSiloPMPIO(file_prefix, file_type, file_postfix,
-                     instance_xtra->mask, t, instance_xtra->file_number,
-                     "Mask");
-      any_file_dumped = 1;
-    }
+      if (public_xtra->write_silopmpio_mask)
+      {
+        strcpy(file_postfix, "");
+        strcpy(file_type, "mask");
+        WriteSiloPMPIO(file_prefix, file_type, file_postfix,
+                       instance_xtra->mask, t, instance_xtra->file_number,
+                       "Mask");
+        any_file_dumped = 1;
+      }
 
-    /* print initial velocities??? jjb */
-    if (print_velocities)
+      /* print initial velocities??? jjb */
+      if (print_velocities)
+      {
+        sprintf(file_postfix, "velx.%05d", instance_xtra->file_number);
+        WritePFBinary(file_prefix, file_postfix,
+                      instance_xtra->x_velocity);
+        static const char* velx_filenames[] = {
+          "velx"
+        };
+        MetadataAddDynamicField(
+                                js_outputs, file_prefix, t, 0, "x-velocity", "m/s", "x-face", "subsurface",
+                                sizeof(velx_filenames) / sizeof(velx_filenames[0]),
+                                velx_filenames);
+
+        sprintf(file_postfix, "vely.%05d", instance_xtra->file_number);
+        WritePFBinary(file_prefix, file_postfix,
+                      instance_xtra->y_velocity);
+        static const char* vely_filenames[] = {
+          "vely"
+        };
+        MetadataAddDynamicField(
+                                js_outputs, file_prefix, t, 0, "y-velocity", "m/s", "y-face", "subsurface",
+                                sizeof(vely_filenames) / sizeof(vely_filenames[0]),
+                                vely_filenames);
+
+        sprintf(file_postfix, "velz.%05d", instance_xtra->file_number);
+        WritePFBinary(file_prefix, file_postfix,
+                      instance_xtra->z_velocity);
+        static const char* velz_filenames[] = {
+          "velz"
+        };
+        MetadataAddDynamicField(
+                                js_outputs, file_prefix, t, 0, "z-velocity", "m/s", "z-face", "subsurface",
+                                sizeof(velz_filenames) / sizeof(velz_filenames[0]),
+                                velz_filenames);
+
+        any_file_dumped = 1;
+      }
+    } // if print_initial_conditions
+    else
     {
-      sprintf(file_postfix, "velx.%05d", instance_xtra->file_number);
-      WritePFBinary(file_prefix, file_postfix,
-                    instance_xtra->x_velocity);
-      static const char* velx_filenames[] = {
-        "velx"
-      };
-      MetadataAddDynamicField(
-                              js_outputs, file_prefix, t, 0, "x-velocity", "m/s", "x-face", "subsurface",
-                              sizeof(velx_filenames) / sizeof(velx_filenames[0]),
-                              velx_filenames);
+      // If not generating initial condition files still bump filenumber using same if conditions
+      // This is not very clean.
+      if (print_wells || print_press || print_satur || public_xtra->print_mask || print_velocities)
+      {
+        any_file_dumped = 1;
+      }
 
-      sprintf(file_postfix, "vely.%05d", instance_xtra->file_number);
-      WritePFBinary(file_prefix, file_postfix,
-                    instance_xtra->y_velocity);
-      static const char* vely_filenames[] = {
-        "vely"
-      };
-      MetadataAddDynamicField(
-                              js_outputs, file_prefix, t, 0, "y-velocity", "m/s", "y-face", "subsurface",
-                              sizeof(vely_filenames) / sizeof(vely_filenames[0]),
-                              vely_filenames);
+      if (public_xtra->write_silo_press || public_xtra->write_silopmpio_press || public_xtra->write_netcdf_press || public_xtra->write_silo_satur || public_xtra->write_silopmpio_satur || public_xtra->write_silo_mask || public_xtra->write_silopmpio_mask)
+      {
+        any_file_dumped = 1;
+      }
 
-      sprintf(file_postfix, "velz.%05d", instance_xtra->file_number);
-      WritePFBinary(file_prefix, file_postfix,
-                    instance_xtra->z_velocity);
-      static const char* velz_filenames[] = {
-        "velz"
-      };
-      MetadataAddDynamicField(
-                              js_outputs, file_prefix, t, 0, "z-velocity", "m/s", "z-face", "subsurface",
-                              sizeof(velz_filenames) / sizeof(velz_filenames[0]),
-                              velz_filenames);
+      if (public_xtra->write_netcdf_press
+          || public_xtra->write_netcdf_satur
+          || public_xtra->write_netcdf_mannings
+          || public_xtra->write_netcdf_subsurface
+          || public_xtra->write_netcdf_slopes
+          || public_xtra->write_netcdf_mask
+          || public_xtra->write_netcdf_dzmult)
+      {
+        any_file_dumped = 1;
+      }
 
-      any_file_dumped = 1;
+      // Need to add metadata
+      if (print_press)
+      {
+        static const char* press_filenames[] = {
+          "press"
+        };
+        MetadataAddDynamicField(
+                                js_outputs, file_prefix, t, 0, "pressure", "m", "cell", "subsurface",
+                                sizeof(press_filenames) / sizeof(press_filenames[0]),
+                                press_filenames);
+      }
+
+      if (print_satur)
+      {
+        static const char* satur_filenames[] = {
+          "satur"
+        };
+        MetadataAddDynamicField(
+                                js_outputs, file_prefix, t, 0, "saturation", NULL, "cell", "subsurface",
+                                sizeof(satur_filenames) / sizeof(satur_filenames[0]),
+                                satur_filenames);
+      }
+
+      if (public_xtra->print_mask)
+      {
+        static const char* mask_filenames[] = {
+          "mask"
+        };
+        MetadataAddStaticField(
+                               js_inputs, file_prefix, "mask", NULL, "cell", "subsurface",
+                               sizeof(mask_filenames) / sizeof(mask_filenames[0]),
+                               mask_filenames);
+      }
+
+      if (print_velocities)
+      {
+        static const char* velx_filenames[] = {
+          "velx"
+        };
+        MetadataAddDynamicField(
+                                js_outputs, file_prefix, t, 0, "x-velocity", "m/s", "x-face", "subsurface",
+                                sizeof(velx_filenames) / sizeof(velx_filenames[0]),
+                                velx_filenames);
+
+        static const char* vely_filenames[] = {
+          "vely"
+        };
+        MetadataAddDynamicField(
+                                js_outputs, file_prefix, t, 0, "y-velocity", "m/s", "y-face", "subsurface",
+                                sizeof(vely_filenames) / sizeof(vely_filenames[0]),
+                                vely_filenames);
+
+        static const char* velz_filenames[] = {
+          "velz"
+        };
+        MetadataAddDynamicField(
+                                js_outputs, file_prefix, t, 0, "z-velocity", "m/s", "z-face", "subsurface",
+                                sizeof(velz_filenames) / sizeof(velz_filenames[0]),
+                                velz_filenames);
+      }
     }
 
     /*-----------------------------------------------------------------
@@ -1547,6 +1679,7 @@ AdvanceRichards(PFModule * this_module, double start_time,      /* Starting time
   int max_iterations = (public_xtra->max_iterations);
   int print_satur = (public_xtra->print_satur);
   int print_wells = (public_xtra->print_wells);
+  int print_reservoirs = (public_xtra->print_reservoirs);
 
   PFModule *problem_saturation = (instance_xtra->problem_saturation);
   PFModule *phase_density = (instance_xtra->phase_density);
@@ -1568,6 +1701,7 @@ AdvanceRichards(PFModule * this_module, double start_time,      /* Starting time
     evap_trans = instance_xtra->evap_trans;
   }
 
+  char filename[2048];          // IMF: 1D input file name *or* 2D/3D input file base name
 #ifdef HAVE_OAS3
   Grid *grid = (instance_xtra->grid);
   Subgrid *subgrid;
@@ -1599,7 +1733,7 @@ AdvanceRichards(PFModule * this_module, double start_time,      /* Starting time
   int ind_veg;                  /*BH: temporary variable to store vegetation index */
   int Stepcount = 0;            /* Added for transient EvapTrans file management - NBE */
   int Loopcount = 0;            /* Added for transient EvapTrans file management - NBE */
-  double sw=NAN, lw=NAN, prcp=NAN, tas=NAN, u=NAN, v=NAN, patm=NAN, qatm=NAN;   // IMF: 1D forcing vars (local to AdvanceRichards)
+  double sw = NAN, lw = NAN, prcp = NAN, tas = NAN, u = NAN, v = NAN, patm = NAN, qatm = NAN;   // IMF: 1D forcing vars (local to AdvanceRichards)
   double lai[18], sai[18], z0m[18], displa[18]; /*BH: array with lai/sai/z0m/displa values for each veg class */
   double *sw_data = NULL;
   double *lw_data = NULL;
@@ -1615,7 +1749,7 @@ AdvanceRichards(PFModule * this_module, double start_time,      /* Starting time
   /*BH*/ double *displa_data = NULL;
   /*BH*/ double *veg_map_data = NULL;
   /*BH*/                        /*will fail if veg_map_data is declared as int */
-  char filename[2048];          // IMF: 1D input file name *or* 2D/3D input file base name
+
   Subvector *sw_forc_sub, *lw_forc_sub, *prcp_forc_sub, *tas_forc_sub, *u_forc_sub, *v_forc_sub, *patm_forc_sub, *qatm_forc_sub, *lai_forc_sub, *sai_forc_sub, *z0m_forc_sub, *displa_forc_sub, *veg_map_forc_sub;      /*BH: added LAI/SAI/Z0M/DISPLA/vegmap */
 
   /* Slopes */
@@ -1624,14 +1758,14 @@ AdvanceRichards(PFModule * this_module, double start_time,      /* Starting time
 
   /* IMF: For writing CLM output */
   Subvector *eflx_lh_tot_sub, *eflx_lwrad_out_sub, *eflx_sh_tot_sub,
-    *eflx_soil_grnd_sub, *qflx_evap_tot_sub, *qflx_evap_grnd_sub,
-    *qflx_evap_soi_sub, *qflx_evap_veg_sub, *qflx_tran_veg_sub,
-    *qflx_infl_sub, *swe_out_sub, *t_grnd_sub, *tsoil_sub, *irr_flag_sub,
-    *qflx_qirr_sub, *qflx_qirr_inst_sub;
+            *eflx_soil_grnd_sub, *qflx_evap_tot_sub, *qflx_evap_grnd_sub,
+            *qflx_evap_soi_sub, *qflx_evap_veg_sub, *qflx_tran_veg_sub,
+            *qflx_infl_sub, *swe_out_sub, *t_grnd_sub, *tsoil_sub, *irr_flag_sub,
+            *qflx_qirr_sub, *qflx_qirr_inst_sub;
 
   double *eflx_lh, *eflx_lwrad, *eflx_sh, *eflx_grnd, *qflx_tot, *qflx_grnd,
-    *qflx_soi, *qflx_eveg, *qflx_tveg, *qflx_in, *swe, *t_g, *t_soi, *iflag,
-    *qirr, *qirr_inst;
+         *qflx_soi, *qflx_eveg, *qflx_tveg, *qflx_in, *swe, *t_g, *t_soi, *iflag,
+         *qirr, *qirr_inst;
   int clm_file_dir_length;
 
   double print_cdt;
@@ -1642,7 +1776,7 @@ AdvanceRichards(PFModule * this_module, double start_time,      /* Starting time
   int any_file_dumped;
   int clm_file_dumped;
   int dump_files = 0;
-  
+
   int retval;
   int converged;
   int take_more_time_steps;
@@ -1747,8 +1881,33 @@ AdvanceRichards(PFModule * this_module, double start_time,      /* Starting time
     {
       ct += cdt;
 
+      // Read in evap_trans even if not HAVE_CLM
+#ifndef HAVE_CLM
+      /******************************************/
+      /*    read transient evap trans flux file */
+      /******************************************/
+      //printf("DEBUG: 2. public_xtra->nc_evap_trans_file_transient %i \n", public_xtra->nc_evap_trans_file_transient);
+      if (public_xtra->nc_evap_trans_file_transient)
+      {
+        strcpy(filename, public_xtra->nc_evap_trans_filename);
+        /*KKu: evaptrans is the name of the variable expected in NetCDF file */
+        /*Here looping similar to pfb is not implemented. All steps are assumed to be
+         * present in the single NetCDF file*/
+        //printf("DEBUG: read evap_trans file %s for step %i \n", filename, instance_xtra->file_number);
+        ReadPFNC(filename, evap_trans, "evaptrans", instance_xtra->file_number, 3);
+        //printf("DEBUG: evap_trans max min: %e %e \n",PFVMax(evap_trans),PFVMin(evap_trans));
+        handle = InitVectorUpdate(evap_trans, VectorUpdateAll);
+        FinalizeVectorUpdate(handle);
+      }
+#endif
       //CPS oasis exchange
 #ifdef HAVE_OAS3
+      // In case `nc_evap_trans_file_transient` is not set in namelist, make
+      // sure `evap_trans` is set to zero at beginning of each loop.
+      if (!public_xtra->nc_evap_trans_file_transient)
+      {
+        InitVectorAll(evap_trans, 0.0);
+      }
       ForSubgridI(is, GridSubgrids(grid))
       {
         int ix, iy, nx, ny, nz, nx_f, ny_f;
@@ -1780,7 +1939,7 @@ AdvanceRichards(PFModule * this_module, double start_time,      /* Starting time
         dz_dat = SubvectorData(dz_sub);
         //CPS       amps_Printf("Calling oasis send/receive for time  %3.1f \n", t);
         CALL_send_fld2_clm(pp, sp, ms, ix, iy, nx, ny, nz, nx_f, ny_f,
-                           t,po_dat,dz_dat);
+                           t, po_dat, dz_dat);
         amps_Sync(amps_CommWorld);
         CALL_receive_fld2_clm(et, ms, ix, iy, nx, ny, nz, nx_f, ny_f, t);
       }
@@ -2433,12 +2592,8 @@ AdvanceRichards(PFModule * this_module, double start_time,      /* Starting time
 
       handle = InitVectorUpdate(evap_trans, VectorUpdateAll);
       FinalizeVectorUpdate(handle);
-
-
-
-
-      //#endif   //End of call to CLM
-
+#endif
+#ifdef HAVE_CLM
       /******************************************/
       /*    read transient evap trans flux file */
       /******************************************/
@@ -2798,10 +2953,10 @@ AdvanceRichards(PFModule * this_module, double start_time,      /* Starting time
         int nx, ny, nz;
         int ip;
         double dx, dy, dz;
-        double vol, vol_max, flux_in, press_pred;
+        double vol, vol_max, flux_in, press_pred, flux_darcy;
 
-        Subvector *p_sub, *s_sub, *et_sub, *po_sub, *dz_sub;
-        double *pp, *sp, *et, *po_dat, *dz_dat;
+        Subvector *p_sub, *s_sub, *et_sub, *po_sub, *dz_sub, *vz_sub, *vx_sub, *vy_sub;
+        double *pp, *sp, *et, *po_dat, *dz_dat, *vz, *vx, *vy;
 
         Subgrid *subgrid;
         Grid *grid = VectorGrid(evap_trans_sum);
@@ -2814,6 +2969,9 @@ AdvanceRichards(PFModule * this_module, double start_time,      /* Starting time
           dz_sub = VectorSubvector(instance_xtra->dz_mult, is);
           s_sub = VectorSubvector(instance_xtra->saturation, is);
           po_sub = VectorSubvector(porosity, is);
+          vx_sub = VectorSubvector(instance_xtra->x_velocity, is);
+          vy_sub = VectorSubvector(instance_xtra->y_velocity, is);
+          vz_sub = VectorSubvector(instance_xtra->z_velocity, is);
 
           r = SubgridRX(subgrid);
 
@@ -2835,31 +2993,54 @@ AdvanceRichards(PFModule * this_module, double start_time,      /* Starting time
           po_dat = SubvectorData(po_sub);
           sp = SubvectorData(s_sub);
 
+          vx = SubvectorData(vx_sub);
+          vy = SubvectorData(vy_sub);
+          vz = SubvectorData(vz_sub);
+
+
           GrGeomInLoop(i, j, k, gr_domain, r, ix, iy, iz, nx, ny, nz,
           {
             ip = SubvectorEltIndex(p_sub, i, j, k);
+            int vxi = SubvectorEltIndex(vx_sub, i + 1, j, k);
+            int vyi = SubvectorEltIndex(vy_sub, i, j + 1, k);
+            int vxi_im1 = SubvectorEltIndex(vx_sub, i, j, k);
+            int vyi_jm1 = SubvectorEltIndex(vy_sub, i, j, k);
+            int vzi_km1 = SubvectorEltIndex(vz_sub, i, j, k);
+
+            int vxi_p1 = SubvectorEltIndex(vx_sub, i + 1, j, k + 1);
+            int vyi_p1 = SubvectorEltIndex(vy_sub, i, j + 1, k + 1);
+            int vxi_im1_p1 = SubvectorEltIndex(vx_sub, i, j, k + 1);
+            int vyi_jm1_p1 = SubvectorEltIndex(vy_sub, i, j, k + 1);
 
             if (k == (nz - 1))
             {
-              vol = dx*dy*dz*dz_dat[ip]*po_dat[ip]*sp[ip];
-              flux_in = dx*dy*dz*dz_dat[ip]*et[ip]*dt;
-              vol_max = dx*dy*dz*dz_dat[ip]*po_dat[ip];
-              press_pred = (flux_in-(vol_max - vol))/(dx*dy*po_dat[ip]);
-              if (flux_in > (vol_max - vol))
+              vol = dx * dy * dz * dz_dat[ip] * po_dat[ip] * sp[ip];
+              flux_in = dx * dy * dz * dz_dat[ip] * et[ip] * dt;
+              vol_max = dx * dy * dz * dz_dat[ip] * po_dat[ip];
+
+              flux_darcy = vz[vzi_km1] * dx * dy * dt + (-vx[vxi] + vx[vxi_im1]) * dy * dz * dz_dat[ip] * dt + (-vy[vyi] + vy[vyi_jm1]) * dx * dz * dz_dat[ip] * dt;
+              press_pred = ((flux_in + flux_darcy) - (vol_max - vol)) / (dx * dy * po_dat[ip]);
+              if ((flux_in + flux_darcy) > (vol_max - vol))
               {
-                if (pp[ip] < 0.0){
+                if (pp[ip] < 0.0)
+                {
+                  if (public_xtra->surface_predictor_pressure > 0.0)
+                  {
+                    press_pred = public_xtra->surface_predictor_pressure;
+                  }
 
-                  press_pred = public_xtra->surface_predictor_pressure;
-                  if (public_xtra->surface_predictor_print == 1) {
-                    amps_Printf(" Cell vol: %3.6e vol_max: %3.6e flux_in: %3.6e  Pressure: %3.6e I: %d J: %d  \n",vol, vol_max,flux_in,pp[ip],i,j);
+                  if (public_xtra->surface_predictor_print == 1)
+                  {
+                    amps_Printf("SP: Cell vol: %3.6e vol_max: %3.6e flux_in: %3.6e  Flux Darcy: %3.6e Cell Pressure: %3.6e Pred Pressure: %3.6e I: %d J: %d  Time: %12.4e  \n", vol, vol_max, flux_in, flux_darcy, pp[ip], press_pred, i, j, t);
+                    amps_Printf("SP: vx_r: %3.6e vx_l: %3.6e vy_r: %3.6e vy_l: %3.6e vz_l: %3.6e  I: %d J: %d k: %d \n", vx[vxi], vx[vxi_im1], vy[vyi], vy[vyi_jm1], vz[vzi_km1], i, j, k);
+                    amps_Printf("SP: vx_r: %3.6e vx_l: %3.6e vy_r: %3.6e vy_l: %3.6e    k+1 \n", vx[vxi_p1], vx[vxi_im1_p1], vy[vyi_p1], vy[vyi_jm1_p1]);
                   }
                   pp[ip] = press_pred;
-
                 }
               }
             }
           }
-        );
+                       );
         }
       }
 
@@ -2946,30 +3127,23 @@ AdvanceRichards(PFModule * this_module, double start_time,      /* Starting time
         GrGeomInLoop(i, j, k, gr_domain, r, ix, iy, iz, nx, ny, nz,
         {
           ip = SubvectorEltIndex(p_sub_sp, i, j, k);
-          // printf(" %d %d %d %d  \n",i,j,k,ip);
-          // printf(" pp[ip] %10.3f \n",pp[ip]);
-          // printf(" NZ: %d \n",nz);
           if (k == (nz - 1))
           {
-            //   printf(" %d %d %d %d  \n",i,j,k,ip);
-            //   printf(" pp[ip] %10.3f \n",pp[ip]);
-
             if (pp_sp[ip] > 0.0)
             {
               printf(" pressure-> 0 %d %d %d %10.3f \n", i, j, k,
                      pp_sp[ip]); pp_sp[ip] = 0.0;
             }
           }
-        }
-                     );
+        });
       }
     }
 
 
-   /***************************************************************
+    /***************************************************************
     *          modify land surface pressures                      *
     ***************************************************************/
-   
+
     if (public_xtra->reset_surface_pressure == 1)
     {
       GrGeomSolid *gr_domain = ProblemDataGrDomain(problem_data);
@@ -3005,29 +3179,153 @@ AdvanceRichards(PFModule * this_module, double start_time,      /* Starting time
         GrGeomInLoop(i, j, k, gr_domain, r, ix, iy, iz, nx, ny, nz,
         {
           ip = SubvectorEltIndex(p_sub_sp, i, j, k);
-          // printf(" %d %d %d %d  \n",i,j,k,ip);
-          // printf(" pp[ip] %10.3f \n",pp[ip]);
-          // printf(" NZ: %d \n",nz);
           if (k == (nz - 1))
           {
-            //   printf(" %d %d %d %d  \n",i,j,k,ip);
-            //   printf(" pp[ip] %10.3f \n",pp[ip]);
-
             if (pp_sp[ip] > public_xtra->threshold_pressure)
             {
-              amps_Printf(" time: %10.3f  pressure reset: %d %d %d %10.3f \n",t, i, j, k,
-                     pp_sp[ip]); pp_sp[ip] = public_xtra->reset_pressure;
+              amps_Printf(" time: %10.3f  pressure reset: %d %d %d %10.3f \n", t, i, j, k,
+                          pp_sp[ip]); pp_sp[ip] = public_xtra->reset_pressure;
             }
           }
         }
                      );
       }
-    /* update pressure,  not sure if we need to do this but we might if pressures are reset along processor edges RMM */
-    handle = InitVectorUpdate(instance_xtra->pressure, VectorUpdateAll);
-    FinalizeVectorUpdate(handle);
+      /* update pressure,  not sure if we need to do this but we might if pressures are reset along processor edges RMM */
+      handle = InitVectorUpdate(instance_xtra->pressure, VectorUpdateAll);
+      FinalizeVectorUpdate(handle);
     }
 
+    ReservoirData         *reservoir_data = ProblemDataReservoirData(problem_data);
+    if (ReservoirDataNumReservoirs(reservoir_data) > 0)
+    {
+      double reservoir_reset_pressure = 0.0;
+      ReservoirDataPhysical *reservoir_data_physical;
+      Subgrid          *tmp_subgrid, *reservoir_intake_subgrid;
+      Subgrid          *reservoir_secondary_intake_subgrid;
+      for (int reservoir = 0; reservoir < ReservoirDataNumReservoirs(reservoir_data); reservoir++)
+      {
+        double flux_in = 0;
+        reservoir_data_physical = ReservoirDataReservoirPhysical(reservoir_data, reservoir);
+        int release_cell_rank;
+        release_cell_rank = ReservoirDataPhysicalReleaseCellMpiRank(reservoir_data_physical);
+        if (release_cell_rank == amps_Rank(amps_CommWorld))
+        {
+          ReservoirDataPhysicalReleaseAmountSinceLastPrint(reservoir_data_physical) +=
+            ReservoirDataPhysicalReleaseAmountInSolver(reservoir_data_physical) * dt;
+          ReservoirDataPhysicalStorage(reservoir_data_physical) -=
+            ReservoirDataPhysicalReleaseAmountInSolver(reservoir_data_physical) * dt;
+        }
+        GrGeomSolid *gr_domain = ProblemDataGrDomain(problem_data);
+
+        int i, j, k, r, is;
+        int ix, iy, iz;
+        int nx, ny, nz;
+        int ip;
+        Subvector *p_sub_sp;
+        double *pp_sp;
 
+        Subgrid *subgrid;
+        Grid *grid = VectorGrid(evap_trans_sum);
+
+        ForSubgridI(is, GridSubgrids(grid))
+        {
+          subgrid = GridSubgrid(grid, is);
+          reservoir_intake_subgrid = ReservoirDataPhysicalIntakeSubgrid(reservoir_data_physical);
+          if ((tmp_subgrid = IntersectSubgrids(subgrid, reservoir_intake_subgrid)))
+          {
+            p_sub_sp = VectorSubvector(instance_xtra->pressure, is);
+
+            r = SubgridRX(tmp_subgrid);
+
+            ix = SubgridIX(tmp_subgrid);
+            iy = SubgridIY(tmp_subgrid);
+            iz = SubgridIZ(tmp_subgrid);
+
+            nx = SubgridNX(tmp_subgrid);
+            ny = SubgridNY(tmp_subgrid);
+            nz = SubgridNZ(tmp_subgrid);
+            double dx = SubgridDX(tmp_subgrid);
+            double dy = SubgridDY(tmp_subgrid);
+            int grid_nz = SubgridNZ(subgrid);
+
+            pp_sp = SubvectorData(p_sub_sp);
+            GrGeomInLoop(i, j, k, gr_domain, r, ix, iy, iz, nx, ny, nz,
+            {
+              ip = SubvectorEltIndex(p_sub_sp, i, j, k);
+              // this assumes a domain where the top of the grid is the same as
+              // the top of the domain
+              if (k == (grid_nz - 1))
+              {
+                if (pp_sp[ip] > reservoir_reset_pressure)
+                {
+                  flux_in += pp_sp[ip] * dx * dy;
+                  pp_sp[ip] = reservoir_reset_pressure;
+                }
+              }
+            }
+                         );
+          }
+        }
+        if (ReservoirDataPhysicalHasSecondaryIntakeCell(reservoir_data_physical))
+        {
+          reservoir_secondary_intake_subgrid = ReservoirDataPhysicalSecondaryIntakeSubgrid(reservoir_data_physical);
+          ForSubgridI(is, GridSubgrids(grid))
+          {
+            subgrid = GridSubgrid(grid, is);
+            if ((tmp_subgrid = IntersectSubgrids(subgrid, reservoir_secondary_intake_subgrid)))
+            {
+              p_sub_sp = VectorSubvector(instance_xtra->pressure, is);
+
+              r = SubgridRX(tmp_subgrid);
+
+              ix = SubgridIX(tmp_subgrid);
+              iy = SubgridIY(tmp_subgrid);
+              iz = SubgridIZ(tmp_subgrid);
+
+              nx = SubgridNX(tmp_subgrid);
+              ny = SubgridNY(tmp_subgrid);
+              nz = SubgridNZ(tmp_subgrid);
+              double dx = SubgridDX(tmp_subgrid);
+              double dy = SubgridDY(tmp_subgrid);
+              int grid_nz = SubgridNZ(subgrid);
+
+              pp_sp = SubvectorData(p_sub_sp);
+              GrGeomInLoop(i, j, k, gr_domain, r, ix, iy, iz, nx, ny, nz,
+              {
+                ip = SubvectorEltIndex(p_sub_sp, i, j, k);
+                // this assumes a domain where the top of the grid is the same as
+                // the top of the domain
+                if (k == (grid_nz - 1))
+                {
+                  if (pp_sp[ip] > reservoir_reset_pressure)
+                  {
+                    flux_in += pp_sp[ip] * dx * dy;
+                    pp_sp[ip] = reservoir_reset_pressure;
+                  }
+                }
+              }
+                           );
+            }
+          }
+        }
+        #ifdef PARFLOW_HAVE_MPI
+        if (ReservoirDataPhysicalMpiCommunicator(reservoir_data_physical) != MPI_COMM_NULL)
+        {
+          amps_Invoice invoice = amps_NewInvoice("%d", &flux_in);
+          amps_AllReduce(ReservoirDataPhysicalMpiCommunicator(reservoir_data_physical), invoice, amps_Add);
+          amps_FreeInvoice(invoice);
+          ReservoirDataPhysicalStorage(reservoir_data_physical) += flux_in;
+          ReservoirDataPhysicalIntakeAmountSinceLastPrint(reservoir_data_physical) += flux_in;
+        }
+        #else
+        ReservoirDataPhysicalStorage(reservoir_data_physical) += flux_in;
+        ReservoirDataPhysicalIntakeAmountSinceLastPrint(reservoir_data_physical) += flux_in;
+        #endif
+        /* update pressure,  not sure if we need to do this but we might if pressures are reset along processor edges RMM */
+      }
+      handle = InitVectorUpdate(instance_xtra->pressure, VectorUpdateAll);
+      FinalizeVectorUpdate(handle);
+    }
 
     /* velocity updates - not sure these are necessary jjb */
     handle = InitVectorUpdate(instance_xtra->x_velocity, VectorUpdateAll);
@@ -3169,7 +3467,6 @@ AdvanceRichards(PFModule * this_module, double start_time,      /* Starting time
                                 "z-velocity", "m/s", "z-face", "subsurface", 0, NULL);
 
         any_file_dumped = 1;
-
       }
 
 
@@ -3800,9 +4097,15 @@ AdvanceRichards(PFModule * this_module, double start_time,      /* Starting time
     }
 
     /*******************************************************************/
-    /*                   Print the Well Data                           */
+    /*                   Print the Well and Reservoir Data                           */
     /*******************************************************************/
-
+    if (print_reservoirs && dump_files)
+    {
+      WriteReservoirs("ReservoirsOutput",
+                      problem,
+                      ProblemDataReservoirData(problem_data),
+                      t, WELLDATA_DONTWRITEHEADER);
+    }
     if (print_wells && dump_files)
     {
       WriteWells(file_prefix,
@@ -3889,10 +4192,10 @@ AdvanceRichards(PFModule * this_module, double start_time,      /* Starting time
       }
     }
 #endif
-    if(first_tstep)
+    if (first_tstep)
     {
       BeginTiming(RichardsExclude1stTimeStepIndex);
-      PUSH_NVTX("RichardsExclude1stTimeStepIndex",6)
+      PUSH_NVTX("RichardsExclude1stTimeStepIndex", 6)
       first_tstep = 0;
     }
   }                             /* ends do for time loop */
@@ -4522,7 +4825,7 @@ SolverRichardsInitInstanceXtra()
 
   /* Compute size for problem parameters */
   parameter_sz = PFModuleSizeOfTempData(instance_xtra->problem_saturation);
-  parameter_sz  += PFModuleSizeOfTempData(instance_xtra->phase_rel_perm);
+  parameter_sz += PFModuleSizeOfTempData(instance_xtra->phase_rel_perm);
 
   /* set temp_data size to max of velocity_sz, concen_sz, and ic_sz. */
   /* The temp vector space for the nonlinear solver is added in because */
@@ -4578,7 +4881,7 @@ SolverRichardsInitInstanceXtra()
   int size_retardation = PFModuleSizeOfTempData(instance_xtra->retardation);
   int size_advect = PFModuleSizeOfTempData(instance_xtra->advect_concen);
   temp_data_placeholder += pfmax(size_retardation, size_advect);
- 
+
   /* set temporary vector data used for advection */
 
   temp_data += temp_data_size;
@@ -4750,22 +5053,6 @@ SolverRichardsNewPublicXtra(char *name)
   sprintf(key, "%s.CLM.CLMFileDir", name);
   public_xtra->clm_file_dir = GetStringDefault(key, "");
 
-  //CPS moved outside CLM def
-  /* @RMM added switch for terrain-following grid */
-  /* RMM set terrain grid (default=False) */
-  //   sprintf(key, "%s.TerrainFollowingGrid", name);
-  //   switch_name = GetStringDefault(key, "False");
-  //   switch_value = NA_NameToIndex(switch_na, switch_name);
-  //   if(switch_value < 0)
-  //   {
-  //       InputError("Error: invalid value <%s> for key <%s>\n",
-  //                  switch_name, key );
-  //   }
-  //   public_xtra -> terrain_following_grid = switch_value;
-
-  //   if (public_xtra -> terrain_following_grid == 1) { printf("TFG true \n");}
-  //CPS
-
   // NBE: Keys for the single file CLM output
   sprintf(key, "%s.CLM.SingleFile", name);
   switch_name = GetStringDefault(key, "False");
@@ -4916,7 +5203,7 @@ SolverRichardsNewPublicXtra(char *name)
   switch_name = GetStringDefault(key, "True");
   switch_value = NA_NameToIndexExitOnError(switch_na, switch_name, key);
   public_xtra->write_CLM_binary = switch_value;
-  
+
 /* IMF Account for slope in CLM energy budget (default=False) */
   sprintf(key, "%s.CLM.UseSlopeAspect", name);
   switch_name = GetStringDefault(key, "False");
@@ -5177,6 +5464,11 @@ SolverRichardsNewPublicXtra(char *name)
   sprintf(key, "%s.DropTol", name);
   public_xtra->drop_tol = GetDoubleDefault(key, 1E-8);
 
+  sprintf(key, "%s.PrintInitialConditions", name);
+  switch_name = GetStringDefault(key, "True");
+  switch_value = NA_NameToIndexExitOnError(switch_na, switch_name, key);
+  public_xtra->print_initial_conditions = switch_value;
+
   sprintf(key, "%s.PrintSubsurfData", name);
   switch_name = GetStringDefault(key, "True");
   switch_value = NA_NameToIndexExitOnError(switch_na, switch_name, key);
@@ -5187,6 +5479,11 @@ SolverRichardsNewPublicXtra(char *name)
   switch_value = NA_NameToIndexExitOnError(switch_na, switch_name, key);
   public_xtra->print_slopes = switch_value;
 
+  sprintf(key, "%s.PrintChannelWidth", name);
+  switch_name = GetStringDefault(key, "False");
+  switch_value = NA_NameToIndexExitOnError(switch_na, switch_name, key);
+  public_xtra->print_channelwidth = switch_value;
+
   sprintf(key, "%s.PrintMannings", name);
   switch_name = GetStringDefault(key, "False");
   switch_value = NA_NameToIndexExitOnError(switch_na, switch_name, key);
@@ -5257,6 +5554,11 @@ SolverRichardsNewPublicXtra(char *name)
   switch_value = NA_NameToIndexExitOnError(switch_na, switch_name, key);
   public_xtra->print_wells = switch_value;
 
+  sprintf(key, "%s.PrintReservoirs", name);
+  switch_name = GetStringDefault(key, "True");
+  switch_value = NA_NameToIndexExitOnError(switch_na, switch_name, key);
+  public_xtra->print_reservoirs = switch_value;
+
   // SGS TODO
   // Need to add this to the user manual, this is new for LSM stuff that was added.
   sprintf(key, "%s.PrintLSMSink", name);
@@ -5636,8 +5938,8 @@ SolverRichardsNewPublicXtra(char *name)
   switch_value = NA_NameToIndexExitOnError(switch_na, switch_name, key);
   public_xtra->spinup = switch_value;
 
- //RMM surface pressure keys
- //Solver.ResetSurfacePressure “True”
+  //RMM surface pressure keys
+  //Solver.ResetSurfacePressure “True”
   sprintf(key, "%s.ResetSurfacePressure", name);
   switch_name = GetStringDefault(key, "False");
   switch_value = NA_NameToIndexExitOnError(switch_na, switch_name, key);
diff --git a/pfsimulator/parflow_lib/stefan_problem_phase_source.c b/pfsimulator/parflow_lib/stefan_problem_phase_source.c
index ca29c97cf..4b37433e7 100644
--- a/pfsimulator/parflow_lib/stefan_problem_phase_source.c
+++ b/pfsimulator/parflow_lib/stefan_problem_phase_source.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -71,7 +71,7 @@ typedef struct {
  *    Phase
  *--------------------------------------------------------------------------*/
 
-void         PhaseSource(phase_source, phase, problem, problem_data, time)
+void PhaseSource(phase_source, phase, problem, problem_data, time)
 
 Vector * phase _source;
 int phase;
@@ -445,15 +445,15 @@ PFModule   *PhaseSourceNewPublicXtra()
         switch_name = GetString(key);
 
         dummy2->geom_indices[ir] = NA_NameToIndexExitOnError(GlobalsGeomNames,
-							     switch_name, key);
+                                                             switch_name, key);
 
         sprintf(key, "Geom.%s.ICPressure.RefPatch", region);
         switch_name = GetString(key);
 
         dummy2->patch_indices[ir] =
           NA_NameToIndexExitOnError(GeomSolidPatches(
-                                          GlobalsGeometries[dummy2->geom_indices[ir]]),
-				    switch_name, key);
+                                                     GlobalsGeometries[dummy2->geom_indices[ir]]),
+                                    switch_name, key);
       }
 
       (public_xtra->data) = (void*)dummy2;
@@ -475,7 +475,8 @@ PFModule   *PhaseSourceNewPublicXtra()
 
     default:
     {
-      InputError("Invalid switch value <%s> for key <%s>", switch_name, key);    }
+      InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+    }
   }
 
   NA_FreeNameArray(type_na);
diff --git a/pfsimulator/parflow_lib/subsrf_sim.c b/pfsimulator/parflow_lib/subsrf_sim.c
index dc108842f..63247de4c 100644
--- a/pfsimulator/parflow_lib/subsrf_sim.c
+++ b/pfsimulator/parflow_lib/subsrf_sim.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *****************************************************************************/
@@ -129,8 +129,8 @@ void SubsrfSim(
   SubgridArray     *subgrids;
 
   Subgrid          *subgrid,
-    *well_subgrid,
-    *tmp_subgrid;
+                   *well_subgrid,
+                   *tmp_subgrid;
 
   Subvector        *perm_x_sub, *perm_y_sub, *perm_z_sub;
   Subvector        *kx_values_sub, *ky_values_sub, *kz_values_sub;
@@ -744,7 +744,7 @@ PFModule   *SubsrfSimNewPublicXtra()
 
       default:
       {
-	InputError("Invalid switch value <%s> for key <%s>", sim_type_name, key);
+        InputError("Invalid switch value <%s> for key <%s>", sim_type_name, key);
       }
     }
   }
@@ -815,7 +815,7 @@ PFModule   *SubsrfSimNewPublicXtra()
 
     default:
     {
-      	InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+      InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
     }
   }   /*End switch */
 
diff --git a/pfsimulator/parflow_lib/time_cycle_data.c b/pfsimulator/parflow_lib/time_cycle_data.c
index caf4ee503..a3968ac2f 100644
--- a/pfsimulator/parflow_lib/time_cycle_data.c
+++ b/pfsimulator/parflow_lib/time_cycle_data.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
diff --git a/pfsimulator/parflow_lib/time_cycle_data.h b/pfsimulator/parflow_lib/time_cycle_data.h
index 0371694f2..98b1e18ab 100644
--- a/pfsimulator/parflow_lib/time_cycle_data.h
+++ b/pfsimulator/parflow_lib/time_cycle_data.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef _TIME_CYCLE_HEADER
 #define _TIME_CYCLE_HEADER
diff --git a/pfsimulator/parflow_lib/timing.c b/pfsimulator/parflow_lib/timing.c
index 42378f9a6..ea0a8c70c 100644
--- a/pfsimulator/parflow_lib/timing.c
+++ b/pfsimulator/parflow_lib/timing.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routines for handling ParFlow timing.
@@ -47,7 +47,7 @@ void  NewTiming()
   timing = ctalloc(TimingType, 1);
 
   /* The order of these registers need to be in sync with the defines
-   * found in solver.h
+   * found in timing.h
    */
 
   RegisterTiming("Solver Setup");
@@ -60,6 +60,7 @@ void  NewTiming()
   RegisterTiming("CLM");
   RegisterTiming("PFSOL Read");
   RegisterTiming("Clustering");
+  RegisterTiming("Netcdf I/O");
 #ifdef VECTOR_UPDATE_TIMING
   RegisterTiming("VectorUpdate");
 #endif
diff --git a/pfsimulator/parflow_lib/timing.h b/pfsimulator/parflow_lib/timing.h
index badaacf1a..9d6abf2bc 100644
--- a/pfsimulator/parflow_lib/timing.h
+++ b/pfsimulator/parflow_lib/timing.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Header file for parflow timing.
@@ -49,8 +49,9 @@
 #define CLMTimingIndex  7
 #define PFSOLReadTimingIndex  8
 #define ClusteringTimingIndex 9
+#define NetcdfTimingIndex 10
 #ifdef VECTOR_UPDATE_TIMING
-#define VectorUpdateTimingIndex  10
+#define VectorUpdateTimingIndex  11
 #endif
 
 
@@ -103,39 +104,39 @@ amps_ThreadLocalDcl(extern TimingType *, timing_ptr);
 
 #define IncFLOPCount(inc) TimingFLOPCount += (FLOPType)inc
 #define StartTiming()     TimingTimeCount -= amps_Clock(); \
-  TimingCPUCount -= amps_CPUClock()
+        TimingCPUCount -= amps_CPUClock()
 #define StopTiming()      TimingTimeCount += amps_Clock(); \
-  TimingCPUCount += amps_CPUClock()
+        TimingCPUCount += amps_CPUClock()
 
 #ifdef TIMING_WITH_SYNC
-#define BeginTiming(i)                  \
-  {                                     \
-    StopTiming();                       \
-    TimingTime(i) -= TimingTimeCount;   \
-    TimingCPUTime(i) -= TimingCPUCount; \
-    TimingFLOPS(i) -= TimingFLOPCount;  \
-    amps_Sync(amps_CommWorld);          \
-    StartTiming();                      \
-  }
+#define BeginTiming(i)                        \
+        {                                     \
+          StopTiming();                       \
+          TimingTime(i) -= TimingTimeCount;   \
+          TimingCPUTime(i) -= TimingCPUCount; \
+          TimingFLOPS(i) -= TimingFLOPCount;  \
+          amps_Sync(amps_CommWorld);          \
+          StartTiming();                      \
+        }
 #else
-#define BeginTiming(i)                  \
-  {                                     \
-    StopTiming();                       \
-    TimingTime(i) -= TimingTimeCount;   \
-    TimingCPUTime(i) -= TimingCPUCount; \
-    TimingFLOPS(i) -= TimingFLOPCount;  \
-    StartTiming();                      \
-  }
+#define BeginTiming(i)                        \
+        {                                     \
+          StopTiming();                       \
+          TimingTime(i) -= TimingTimeCount;   \
+          TimingCPUTime(i) -= TimingCPUCount; \
+          TimingFLOPS(i) -= TimingFLOPCount;  \
+          StartTiming();                      \
+        }
 #endif
 
-#define EndTiming(i)                    \
-  {                                     \
-    StopTiming();                       \
-    TimingTime(i) += TimingTimeCount;   \
-    TimingCPUTime(i) += TimingCPUCount; \
-    TimingFLOPS(i) += TimingFLOPCount;  \
-    StartTiming();                      \
-  }
+#define EndTiming(i)                          \
+        {                                     \
+          StopTiming();                       \
+          TimingTime(i) += TimingTimeCount;   \
+          TimingCPUTime(i) += TimingCPUCount; \
+          TimingFLOPS(i) += TimingFLOPCount;  \
+          StartTiming();                      \
+        }
 
 #ifdef VECTOR_UPDATE_TIMING
 
diff --git a/pfsimulator/parflow_lib/total_velocity_face.c b/pfsimulator/parflow_lib/total_velocity_face.c
index ff82aab89..9a8a23813 100644
--- a/pfsimulator/parflow_lib/total_velocity_face.c
+++ b/pfsimulator/parflow_lib/total_velocity_face.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 *****************************************************************************/
@@ -33,7 +33,7 @@
 
 #define Func(a, b) HarmonicMean(a, b)
 #define Coeff(Ta, Tb, Pa, Pb) \
-  (((Ta) + (Tb)) ? ((Ta) * (Pb) + (Tb) * (Pa)) / ((Ta) + (Tb)) : 0)
+        (((Ta) + (Tb)) ? ((Ta) * (Pb) + (Tb) * (Pa)) / ((Ta) + (Tb)) : 0)
 
 #define FuncOps     4
 #define CoeffOps    5
@@ -660,18 +660,18 @@ void    TotalVelocityFace(
                               j + dir[dir0][1],
                               k + dir[dir0][2]);
 
-	/* 
-	   vel1_l = SubvectorElt(subvector_v1, i, j, k);
-	   vel1_r = SubvectorElt(subvector_v1,
-	   i + dir[dir1][0],
-	   j + dir[dir1][1],
-	   k + dir[dir1][2]);
-	   vel2_l = SubvectorElt(subvector_v2, i, j, k);
-	   vel2_r = SubvectorElt(subvector_v2,
-	   i + dir[dir2][0],
-	   j + dir[dir2][1],
-	   k + dir[dir2][2]);
-	*/
+        /*
+         * vel1_l = SubvectorElt(subvector_v1, i, j, k);
+         * vel1_r = SubvectorElt(subvector_v1,
+         * i + dir[dir1][0],
+         * j + dir[dir1][1],
+         * k + dir[dir1][2]);
+         * vel2_l = SubvectorElt(subvector_v2, i, j, k);
+         * vel2_r = SubvectorElt(subvector_v2,
+         * i + dir[dir2][0],
+         * j + dir[dir2][1],
+         * k + dir[dir2][2]);
+         */
 
         if (fdir[dir0] == -1)
         {
@@ -684,9 +684,9 @@ void    TotalVelocityFace(
         vel0_r[0] = 0.0;
 
         /*
-	 * h0 = ds[dir0];
-	 * h1 = ds[dir1];
-	 * h2 = ds[dir2];
+         * h0 = ds[dir0];
+         * h1 = ds[dir1];
+         * h2 = ds[dir2];
          * alpha = -fdir[dir0];
          * vel0_r[0] = vel0_l[0]
          + alpha*h0*( (vel1_r[0] - vel1_l[0])/h1
@@ -872,7 +872,7 @@ void TotalVelocityFaceFreePublicXtra()
 
   if (public_xtra)
   {
-    free(public_xtra);
+    tfree(public_xtra);
   }
 }
 
diff --git a/pfsimulator/parflow_lib/turning_bands.c b/pfsimulator/parflow_lib/turning_bands.c
index 314e1a8fc..48a3d5337 100644
--- a/pfsimulator/parflow_lib/turning_bands.c
+++ b/pfsimulator/parflow_lib/turning_bands.c
@@ -94,7 +94,7 @@ typedef struct {
  * Turn
  *--------------------------------------------------------------------------*/
 
-void    Turn(field, vxtra)
+void Turn(field, vxtra)
 Vector * field;
 void   *vxtra;
 {
@@ -370,7 +370,7 @@ char *geom_name;
   else
   {
     NA_InputError("Error: Invalid True/False value for key <%s> was <%s>\n",
-               key, tmp);
+                  key, tmp);
   }
 
   sprintf(key, "Geom.%s.Perm.Seed", geom_name);
@@ -388,7 +388,7 @@ char *geom_name;
  * FreeTurn
  *--------------------------------------------------------------------------*/
 
-void           FreeTurn(xtra)
+void FreeTurn(xtra)
 void *xtra;
 {
   free(xtra);
diff --git a/pfsimulator/parflow_lib/turning_bandsRF.c b/pfsimulator/parflow_lib/turning_bandsRF.c
index 066d4dd6e..5542128c2 100644
--- a/pfsimulator/parflow_lib/turning_bandsRF.c
+++ b/pfsimulator/parflow_lib/turning_bandsRF.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routines to generate a Gaussian random field.
diff --git a/pfsimulator/parflow_lib/usergrid_input.c b/pfsimulator/parflow_lib/usergrid_input.c
index 9af8b860e..784118445 100644
--- a/pfsimulator/parflow_lib/usergrid_input.c
+++ b/pfsimulator/parflow_lib/usergrid_input.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * ReadUserSubgrid, ReadUserGrid,
 * FreeUserGrid
diff --git a/pfsimulator/parflow_lib/vector.c b/pfsimulator/parflow_lib/vector.c
index 7414e6703..8694376b5 100644
--- a/pfsimulator/parflow_lib/vector.c
+++ b/pfsimulator/parflow_lib/vector.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routines for manipulating vector structures.
@@ -244,7 +244,7 @@ static Vector  *NewTempVector(
   memset(new_vector, 0, sizeof(Vector));
 
   (new_vector->subvectors) = talloc(Subvector *, GridNumSubgrids(grid));    /* 1st arg.: variable type;
-                                                                              * 2nd arg.: # of elements to be allocated*/
+                                                                             * 2nd arg.: # of elements to be allocated*/
   memset(new_vector->subvectors, 0, GridNumSubgrids(grid) * sizeof(Subvector *));
 
   data_size = 0;
@@ -348,7 +348,7 @@ Vector  *NewVectorType(
 
 #ifdef HAVE_SAMRAI
   enum ParflowGridType grid_type = invalid_grid_type;
-  
+
   switch (type)
   {
     case vector_cell_centered:
diff --git a/pfsimulator/parflow_lib/vector.h b/pfsimulator/parflow_lib/vector.h
index 6426946ca..76248d989 100644
--- a/pfsimulator/parflow_lib/vector.h
+++ b/pfsimulator/parflow_lib/vector.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Header info for the Vector data structures
@@ -128,15 +128,15 @@ typedef struct _VectorUpdateCommHandle {
 #define SubvectorNY(subvector)   (SubgridNY(SubvectorDataSpace(subvector)))
 #define SubvectorNZ(subvector)   (SubgridNZ(SubvectorDataSpace(subvector)))
 
-#define SubvectorEltIndex(subvector, x, y, z) \
-  (((x) - SubvectorIX(subvector)) +           \
-   (((y) - SubvectorIY(subvector)) +          \
-    (((z) - SubvectorIZ(subvector))) *        \
-    SubvectorNY(subvector)) *                 \
-   SubvectorNX(subvector))
+#define SubvectorEltIndex(subvector, x, y, z)       \
+        (((x) - SubvectorIX(subvector)) +           \
+         (((y) - SubvectorIY(subvector)) +          \
+          (((z) - SubvectorIZ(subvector))) *        \
+          SubvectorNY(subvector)) *                 \
+         SubvectorNX(subvector))
 
 #define SubvectorElt(subvector, x, y, z) \
-  (SubvectorData(subvector) + SubvectorEltIndex(subvector, x, y, z))
+        (SubvectorData(subvector) + SubvectorEltIndex(subvector, x, y, z))
 
 #define SubvectorDataSize(subvector) ((subvector)->data_size)
 
diff --git a/pfsimulator/parflow_lib/vector_utilities.c b/pfsimulator/parflow_lib/vector_utilities.c
index 97c0118ee..2ca47b781 100644
--- a/pfsimulator/parflow_lib/vector_utilities.c
+++ b/pfsimulator/parflow_lib/vector_utilities.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /* This file contains utility routines for ParFlow's Vector class.
  *
  * This file was modified from a coresponding PVODE package file to account
@@ -723,9 +723,9 @@ double PFVDotProd(
     i_y = 0;
 
     BoxLoopReduceI2(sum,
-              i, j, k, ix, iy, iz, nx, ny, nz,
-              i_x, nx_x, ny_x, nz_x, 1, 1, 1,
-              i_y, nx_y, ny_y, nz_y, 1, 1, 1,
+                    i, j, k, ix, iy, iz, nx, ny, nz,
+                    i_x, nx_x, ny_x, nz_x, 1, 1, 1,
+                    i_y, nx_y, ny_y, nz_y, 1, 1, 1,
     {
       ReduceSum(sum, xp[i_x] * yp[i_y]);
     });
@@ -782,8 +782,8 @@ double PFVMaxNorm(
 
     i_x = 0;
     BoxLoopReduceI1(max_val,
-              i, j, k, ix, iy, iz, nx, ny, nz,
-              i_x, nx_x, ny_x, nz_x, 1, 1, 1,
+                    i, j, k, ix, iy, iz, nx, ny, nz,
+                    i_x, nx_x, ny_x, nz_x, 1, 1, 1,
     {
       double xp_abs = fabs(xp[i_x]);
       ReduceMax(max_val, xp_abs);
@@ -851,9 +851,9 @@ double PFVWrmsNorm(
     i_w = 0;
 
     BoxLoopReduceI2(sum,
-              i, j, k, ix, iy, iz, nx, ny, nz,
-              i_x, nx_x, ny_x, nz_x, 1, 1, 1,
-              i_w, nx_w, ny_w, nz_w, 1, 1, 1,
+                    i, j, k, ix, iy, iz, nx, ny, nz,
+                    i_x, nx_x, ny_x, nz_x, 1, 1, 1,
+                    i_w, nx_w, ny_w, nz_w, 1, 1, 1,
     {
       double prod = xp[i_x] * wp[i_w];
       ReduceSum(sum, prod * prod);
@@ -923,9 +923,9 @@ double PFVWL2Norm(
     i_w = 0;
 
     BoxLoopReduceI2(sum,
-              i, j, k, ix, iy, iz, nx, ny, nz,
-              i_x, nx_x, ny_x, nz_x, 1, 1, 1,
-              i_w, nx_w, ny_w, nz_w, 1, 1, 1,
+                    i, j, k, ix, iy, iz, nx, ny, nz,
+                    i_x, nx_x, ny_x, nz_x, 1, 1, 1,
+                    i_w, nx_w, ny_w, nz_w, 1, 1, 1,
     {
       const double prod = xp[i_x] * wp[i_w];
       ReduceSum(sum, prod * prod);
@@ -983,8 +983,8 @@ double PFVL1Norm(
 
     i_x = 0;
     BoxLoopReduceI1(sum,
-              i, j, k, ix, iy, iz, nx, ny, nz,
-              i_x, nx_x, ny_x, nz_x, 1, 1, 1,
+                    i, j, k, ix, iy, iz, nx, ny, nz,
+                    i_x, nx_x, ny_x, nz_x, 1, 1, 1,
     {
       ReduceSum(sum, fabs(xp[i_x]));
     });
@@ -1046,8 +1046,8 @@ double PFVMin(
     {
       i_x = 0;
       BoxLoopReduceI1(min_val,
-                i, j, k, ix, iy, iz, 1, 1, 1,
-                i_x, nx_x, ny_x, nz_x, 1, 1, 1,
+                      i, j, k, ix, iy, iz, 1, 1, 1,
+                      i_x, nx_x, ny_x, nz_x, 1, 1, 1,
       {
         ReduceSum(min_val, xp[i_x]);
       });
@@ -1055,8 +1055,8 @@ double PFVMin(
 
     i_x = 0;
     BoxLoopReduceI1(min_val,
-              i, j, k, ix, iy, iz, nx, ny, nz,
-              i_x, nx_x, ny_x, nz_x, 1, 1, 1,
+                    i, j, k, ix, iy, iz, nx, ny, nz,
+                    i_x, nx_x, ny_x, nz_x, 1, 1, 1,
     {
       ReduceMin(min_val, xp[i_x]);
     });
@@ -1113,8 +1113,8 @@ double PFVMax(
     {
       i_x = 0;
       BoxLoopReduceI1(max_val,
-                i, j, k, ix, iy, iz, 1, 1, 1,
-                i_x, nx_x, ny_x, nz_x, 1, 1, 1,
+                      i, j, k, ix, iy, iz, 1, 1, 1,
+                      i_x, nx_x, ny_x, nz_x, 1, 1, 1,
       {
         ReduceSum(max_val, xp[i_x]);
       });
@@ -1123,8 +1123,8 @@ double PFVMax(
     i_x = 0;
 
     BoxLoopReduceI1(max_val,
-              i, j, k, ix, iy, iz, nx, ny, nz,
-              i_x, nx_x, ny_x, nz_x, 1, 1, 1,
+                    i, j, k, ix, iy, iz, nx, ny, nz,
+                    i_x, nx_x, ny_x, nz_x, 1, 1, 1,
     {
       ReduceMax(max_val, xp[i_x]);
     });
@@ -1385,7 +1385,7 @@ void PFVCopy(Vector *x,
     Subvector  *x_sub = VectorSubvector(x, sg);
     Subvector  *y_sub = VectorSubvector(y, sg);
 
-    tmemcpy(SubvectorData(y_sub), SubvectorData(x_sub), SubvectorDataSize(y_sub)*sizeof(double));
+    tmemcpy(SubvectorData(y_sub), SubvectorData(x_sub), SubvectorDataSize(y_sub) * sizeof(double));
   }
 }
 
diff --git a/pfsimulator/parflow_lib/w_jacobi.c b/pfsimulator/parflow_lib/w_jacobi.c
index 7f57c2dd7..266a1af50 100644
--- a/pfsimulator/parflow_lib/w_jacobi.c
+++ b/pfsimulator/parflow_lib/w_jacobi.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Weighted Jacobi iteration.
diff --git a/pfsimulator/parflow_lib/well.c b/pfsimulator/parflow_lib/well.c
index 6a29e1367..6bfe0880a 100644
--- a/pfsimulator/parflow_lib/well.c
+++ b/pfsimulator/parflow_lib/well.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
diff --git a/pfsimulator/parflow_lib/well.h b/pfsimulator/parflow_lib/well.h
index e14a7cecf..136c16a21 100644
--- a/pfsimulator/parflow_lib/well.h
+++ b/pfsimulator/parflow_lib/well.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef _WELL_HEADER
 #define _WELL_HEADER
@@ -107,124 +107,124 @@ typedef struct {
  * Accessor macros: WellDataPhysical
  *--------------------------------------------------------------------------*/
 #define WellDataPhysicalNumber(well_data_physical) \
-  ((well_data_physical)->number)
+        ((well_data_physical)->number)
 
 #define WellDataPhysicalName(well_data_physical) \
-  ((well_data_physical)->name)
+        ((well_data_physical)->name)
 
 #define WellDataPhysicalXLower(well_data_physical) \
-  ((well_data_physical)->x_lower)
+        ((well_data_physical)->x_lower)
 
 #define WellDataPhysicalYLower(well_data_physical) \
-  ((well_data_physical)->y_lower)
+        ((well_data_physical)->y_lower)
 
 #define WellDataPhysicalZLower(well_data_physical) \
-  ((well_data_physical)->z_lower)
+        ((well_data_physical)->z_lower)
 
 #define WellDataPhysicalXUpper(well_data_physical) \
-  ((well_data_physical)->x_upper)
+        ((well_data_physical)->x_upper)
 
 #define WellDataPhysicalYUpper(well_data_physical) \
-  ((well_data_physical)->y_upper)
+        ((well_data_physical)->y_upper)
 
 #define WellDataPhysicalZUpper(well_data_physical) \
-  ((well_data_physical)->z_upper)
+        ((well_data_physical)->z_upper)
 
 #define WellDataPhysicalSubgrid(well_data_physical) \
-  ((well_data_physical)->subgrid)
+        ((well_data_physical)->subgrid)
 
 #define WellDataPhysicalDiameter(well_data_physical) \
-  ((well_data_physical)->diameter)
+        ((well_data_physical)->diameter)
 
 #define WellDataPhysicalSize(well_data_physical) \
-  ((well_data_physical)->size)
+        ((well_data_physical)->size)
 
 #define WellDataPhysicalAction(well_data_physical) \
-  ((well_data_physical)->action)
+        ((well_data_physical)->action)
 
 #define WellDataPhysicalMethod(well_data_physical) \
-  ((well_data_physical)->method)
+        ((well_data_physical)->method)
 
 #define WellDataPhysicalCycleNumber(well_data_physical) \
-  ((well_data_physical)->cycle_number)
+        ((well_data_physical)->cycle_number)
 
 #define WellDataPhysicalAveragePermeabilityX(well_data_physical) \
-  ((well_data_physical)->average_permeability_x)
+        ((well_data_physical)->average_permeability_x)
 
 #define WellDataPhysicalAveragePermeabilityY(well_data_physical) \
-  ((well_data_physical)->average_permeability_y)
+        ((well_data_physical)->average_permeability_y)
 
 #define WellDataPhysicalAveragePermeabilityZ(well_data_physical) \
-  ((well_data_physical)->average_permeability_z)
+        ((well_data_physical)->average_permeability_z)
 
 /*--------------------------------------------------------------------------
  * Accessor macros: WellDataValue
  *--------------------------------------------------------------------------*/
 #define WellDataValueIntervalValues(well_data_values) \
-  (((well_data)->press_well_values[i])[interval_number])
+        (((well_data)->press_well_values[i])[interval_number])
 
 #define WellDataValuePhaseValues(well_data_value) \
-  ((well_data_value)->phase_values)
+        ((well_data_value)->phase_values)
 #define WellDataValuePhaseValue(well_data_value, i) \
-  ((well_data_value)->phase_values[i])
+        ((well_data_value)->phase_values[i])
 
 #define WellDataValueSaturationValues(well_data_value) \
-  ((well_data_value)->saturation_values)
+        ((well_data_value)->saturation_values)
 #define WellDataValueSaturationValue(well_data_value, i) \
-  ((well_data_value)->saturation_values[i])
+        ((well_data_value)->saturation_values[i])
 
 #define WellDataValueDeltaSaturationPtrs(well_data_value) \
-  ((well_data_value)->delta_saturation_ptrs)
+        ((well_data_value)->delta_saturation_ptrs)
 #define WellDataValueDeltaSaturationPtr(well_data_value, i) \
-  ((well_data_value)->delta_saturation_ptrs[i])
+        ((well_data_value)->delta_saturation_ptrs[i])
 
 #define WellDataValueContaminantValues(well_data_value) \
-  ((well_data_value)->contaminant_values)
+        ((well_data_value)->contaminant_values)
 #define WellDataValueContaminantValue(well_data_value, i) \
-  ((well_data_value)->contaminant_values[i])
+        ((well_data_value)->contaminant_values[i])
 
 #define WellDataValueDeltaContaminantPtrs(well_data_value) \
-  ((well_data_value)->delta_contaminant_ptrs)
+        ((well_data_value)->delta_contaminant_ptrs)
 #define WellDataValueDeltaContaminantPtr(well_data_value, i) \
-  ((well_data_value)->delta_contaminant_ptrs[i])
+        ((well_data_value)->delta_contaminant_ptrs[i])
 
 #define WellDataValueContaminantFractions(well_data_value) \
-  ((well_data_value)->contaminant_fractions)
+        ((well_data_value)->contaminant_fractions)
 #define WellDataValueContaminantFraction(well_data_value, i) \
-  ((well_data_value)->contaminant_fractions[i])
+        ((well_data_value)->contaminant_fractions[i])
 
 /*--------------------------------------------------------------------------
  * Accessor macros: WellDataStat
  *--------------------------------------------------------------------------*/
 #define WellDataStatDeltaPhases(well_data_stat) \
-  ((well_data_stat)->delta_phases)
+        ((well_data_stat)->delta_phases)
 #define WellDataStatDeltaPhase(well_data_stat, i) \
-  ((well_data_stat)->delta_phases[i])
+        ((well_data_stat)->delta_phases[i])
 
 #define WellDataStatPhaseStats(well_data_stat) \
-  ((well_data_stat)->phase_stats)
+        ((well_data_stat)->phase_stats)
 #define WellDataStatPhaseStat(well_data_stat, i) \
-  ((well_data_stat)->phase_stats[i])
+        ((well_data_stat)->phase_stats[i])
 
 #define WellDataStatDeltaSaturations(well_data_stat) \
-  ((well_data_stat)->delta_saturations)
+        ((well_data_stat)->delta_saturations)
 #define WellDataStatDeltaSaturation(well_data_stat, i) \
-  ((well_data_stat)->delta_saturations[i])
+        ((well_data_stat)->delta_saturations[i])
 
 #define WellDataStatSaturationStats(well_data_stat) \
-  ((well_data_stat)->saturation_stats)
+        ((well_data_stat)->saturation_stats)
 #define WellDataStatSaturationStat(well_data_stat, i) \
-  ((well_data_stat)->saturation_stats[i])
+        ((well_data_stat)->saturation_stats[i])
 
 #define WellDataStatDeltaContaminants(well_data_stat) \
-  ((well_data_stat)->delta_contaminants)
+        ((well_data_stat)->delta_contaminants)
 #define WellDataStatDeltaContaminant(well_data_stat, i) \
-  ((well_data_stat)->delta_contaminants[i])
+        ((well_data_stat)->delta_contaminants[i])
 
 #define WellDataStatContaminantStats(well_data_stat) \
-  ((well_data_stat)->contaminant_stats)
+        ((well_data_stat)->contaminant_stats)
 #define WellDataStatContaminantStat(well_data_stat, i) \
-  ((well_data_stat)->contaminant_stats[i])
+        ((well_data_stat)->contaminant_stats[i])
 
 /*--------------------------------------------------------------------------
  * Accessor macros: WellData
@@ -240,41 +240,41 @@ typedef struct {
 #define WellDataNumPressWells(well_data) ((well_data)->num_press_wells)
 
 #define WellDataPressWellPhysicals(well_data) \
-  ((well_data)->press_well_physicals)
+        ((well_data)->press_well_physicals)
 #define WellDataPressWellPhysical(well_data, i) \
-  ((well_data)->press_well_physicals[i])
+        ((well_data)->press_well_physicals[i])
 
 #define WellDataPressWellValues(well_data) \
-  ((well_data)->press_well_values)
+        ((well_data)->press_well_values)
 #define WellDataPressWellIntervalValues(well_data, i) \
-  ((well_data)->press_well_values[i])
+        ((well_data)->press_well_values[i])
 #define WellDataPressWellIntervalValue(well_data, i, interval_number) \
-  (((well_data)->press_well_values[i])[interval_number])
+        (((well_data)->press_well_values[i])[interval_number])
 
 #define WellDataPressWellStats(well_data) \
-  ((well_data)->press_well_stats)
+        ((well_data)->press_well_stats)
 #define WellDataPressWellStat(well_data, i) \
-  ((well_data)->press_well_stats[i])
+        ((well_data)->press_well_stats[i])
 
 /*---------------------------- Flux well data ------------------------------*/
 #define WellDataNumFluxWells(well_data)      ((well_data)->num_flux_wells)
 
 #define WellDataFluxWellPhysicals(well_data) \
-  ((well_data)->flux_well_physicals)
+        ((well_data)->flux_well_physicals)
 #define WellDataFluxWellPhysical(well_data, i) \
-  ((well_data)->flux_well_physicals[i])
+        ((well_data)->flux_well_physicals[i])
 
 #define WellDataFluxWellValues(well_data) \
-  ((well_data)->flux_well_values)
+        ((well_data)->flux_well_values)
 #define WellDataFluxWellIntervalValues(well_data, i) \
-  ((well_data)->flux_well_values[i])
+        ((well_data)->flux_well_values[i])
 #define WellDataFluxWellIntervalValue(well_data, i, interval_number) \
-  (((well_data)->flux_well_values[i])[interval_number])
+        (((well_data)->flux_well_values[i])[interval_number])
 
 #define WellDataFluxWellStats(well_data) \
-  ((well_data)->flux_well_stats)
+        ((well_data)->flux_well_stats)
 #define WellDataFluxWellStat(well_data, i) \
-  ((well_data)->flux_well_stats[i])
+        ((well_data)->flux_well_stats[i])
 
 /*--------------------------------------------------------------------------
  * Well Data constants used in the program.
diff --git a/pfsimulator/parflow_lib/well_package.c b/pfsimulator/parflow_lib/well_package.c
index 07c044290..de6b28054 100644
--- a/pfsimulator/parflow_lib/well_package.c
+++ b/pfsimulator/parflow_lib/well_package.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "parflow.h"
 
@@ -128,7 +128,7 @@ void         WellPackage(
   double          **phase_values;
   double subgrid_volume;
   double x_lower, x_upper, y_lower, y_upper,
-    z_lower, z_upper;
+         z_lower, z_upper;
 
   /* Allocate the well data */
   WellDataNumPhases(well_data) = (public_xtra->num_phases);
@@ -865,7 +865,8 @@ PFModule  *WellPackageNewPublicXtra(
   (public_xtra->num_phases) = num_phases;
   (public_xtra->num_contaminants) = num_contaminants;
 
-  well_names = GetString("Wells.Names");
+  char* EMPTY_NAMES_LIST = "";
+  well_names = GetStringDefault("Wells.Names", EMPTY_NAMES_LIST);
 
   public_xtra->well_names = NA_NewNameArray(well_names);
 
@@ -1283,7 +1284,7 @@ PFModule  *WellPackageNewPublicXtra(
 
         default:
         {
-	  InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
+          InputError("Invalid switch value <%s> for key <%s>", switch_name, key);
         }
       }
     }
diff --git a/pfsimulator/parflow_lib/wells_lb.c b/pfsimulator/parflow_lib/wells_lb.c
index 310094827..31ba82137 100644
--- a/pfsimulator/parflow_lib/wells_lb.c
+++ b/pfsimulator/parflow_lib/wells_lb.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 *
diff --git a/pfsimulator/parflow_lib/wrf_parflow.c b/pfsimulator/parflow_lib/wrf_parflow.c
index f010b90bf..a3f7f25d7 100644
--- a/pfsimulator/parflow_lib/wrf_parflow.c
+++ b/pfsimulator/parflow_lib/wrf_parflow.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routine to be invoked by WRF model.
diff --git a/pfsimulator/parflow_lib/write_clm_netcdf.c b/pfsimulator/parflow_lib/write_clm_netcdf.c
index 074723162..cc707d627 100644
--- a/pfsimulator/parflow_lib/write_clm_netcdf.c
+++ b/pfsimulator/parflow_lib/write_clm_netcdf.c
@@ -1,29 +1,35 @@
-/*BHEADER*********************************************************************
- *
- *  This file is part of Parflow. For details, see
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
- *
- * Routines to write a Vector to a file in full or scattered form.
- *
- *****************************************************************************/
+*
+* Routines to write a Vector to a file in full or scattered form.
+*
+*****************************************************************************/
 
 #include "parflow.h"
 #include "parflow_netcdf.h"
@@ -45,6 +51,7 @@ void WriteCLMNC(char * file_prefix, char* file_postfix, double t, Vector  *v, in
                 char *varName, int dimensionality)
 {
 #ifdef PARFLOW_HAVE_NETCDF
+  BeginTiming(NetcdfTimingIndex);
   static int numCLMStepsInFile = 0;
   int userSpecSteps = GetInt("NetCDF.CLMNumStepsPerFile");
   static char file_name[255];
@@ -90,6 +97,7 @@ void WriteCLMNC(char * file_prefix, char* file_postfix, double t, Vector  *v, in
       }
     }
   }
+  EndTiming(NetcdfTimingIndex);
 #else
   amps_Printf("Parflow not compiled with NetCDF, can't create NetCDF file\n");
 #endif
@@ -101,7 +109,7 @@ void CreateCLMNCFile(char *file_name, int *clmIDs)
   char *switch_name;
   char key[IDB_MAX_KEY_LEN];
   char *default_val = "None";
-  
+
   sprintf(key, "NetCDF.ROMIOhints");
   switch_name = GetStringDefault(key, "None");
   if (strcmp(switch_name, default_val) != 0)
@@ -255,8 +263,9 @@ int LookUpCLMInventory(char * varName, varNCData **myVarNCData, int *clmIDs)
         chunksize[3] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(clmIDs[0], tsoilCLMVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(clmIDs[0],tsoilCLMVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(clmIDs[0], tsoilCLMVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -294,8 +303,9 @@ int LookUpCLMInventory(char * varName, varNCData **myVarNCData, int *clmIDs)
         chunksize[2] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(clmIDs[0], lhTotVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(clmIDs[0],lhTotVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(clmIDs[0], lhTotVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -333,8 +343,9 @@ int LookUpCLMInventory(char * varName, varNCData **myVarNCData, int *clmIDs)
         chunksize[2] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(clmIDs[0], lwradVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(clmIDs[0],lwradVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(clmIDs[0], lwradVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -372,8 +383,9 @@ int LookUpCLMInventory(char * varName, varNCData **myVarNCData, int *clmIDs)
         chunksize[2] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(clmIDs[0], shTotVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(clmIDs[0],shTotVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(clmIDs[0], shTotVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -411,8 +423,9 @@ int LookUpCLMInventory(char * varName, varNCData **myVarNCData, int *clmIDs)
         chunksize[2] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(clmIDs[0], soilGrndVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(clmIDs[0],soilGrndVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(clmIDs[0], soilGrndVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -450,8 +463,9 @@ int LookUpCLMInventory(char * varName, varNCData **myVarNCData, int *clmIDs)
         chunksize[2] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(clmIDs[0], qEvapTotVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(clmIDs[0],qEvapTotVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(clmIDs[0], qEvapTotVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -489,8 +503,9 @@ int LookUpCLMInventory(char * varName, varNCData **myVarNCData, int *clmIDs)
         chunksize[2] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(clmIDs[0], qEvapGrndVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(clmIDs[0],qEvapGrndVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(clmIDs[0], qEvapGrndVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -528,8 +543,9 @@ int LookUpCLMInventory(char * varName, varNCData **myVarNCData, int *clmIDs)
         chunksize[2] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(clmIDs[0], qEvapSoiVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(clmIDs[0],qEvapSoiVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(clmIDs[0], qEvapSoiVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -567,8 +583,9 @@ int LookUpCLMInventory(char * varName, varNCData **myVarNCData, int *clmIDs)
         chunksize[2] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(clmIDs[0], qEvapVegVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(clmIDs[0],qEvapVegVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(clmIDs[0], qEvapVegVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -606,8 +623,9 @@ int LookUpCLMInventory(char * varName, varNCData **myVarNCData, int *clmIDs)
         chunksize[2] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(clmIDs[0], qTranVegVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(clmIDs[0],qTranVegVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(clmIDs[0], qTranVegVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -645,8 +663,9 @@ int LookUpCLMInventory(char * varName, varNCData **myVarNCData, int *clmIDs)
         chunksize[2] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(clmIDs[0], qInflVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(clmIDs[0],qInflVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(clmIDs[0], qInflVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -684,8 +703,9 @@ int LookUpCLMInventory(char * varName, varNCData **myVarNCData, int *clmIDs)
         chunksize[2] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(clmIDs[0], sweVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(clmIDs[0],sweVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(clmIDs[0], sweVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -723,8 +743,9 @@ int LookUpCLMInventory(char * varName, varNCData **myVarNCData, int *clmIDs)
         chunksize[2] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(clmIDs[0], t_grndVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(clmIDs[0],t_grndVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(clmIDs[0], t_grndVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -762,8 +783,9 @@ int LookUpCLMInventory(char * varName, varNCData **myVarNCData, int *clmIDs)
         chunksize[2] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(clmIDs[0], qQirrVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(clmIDs[0],qQirrVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(clmIDs[0], qQirrVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -803,8 +825,9 @@ int LookUpCLMInventory(char * varName, varNCData **myVarNCData, int *clmIDs)
         chunksize[3] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(clmIDs[0], qQirrInstCLMVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(clmIDs[0],qQirrInstCLMVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(clmIDs[0], qQirrInstCLMVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -816,7 +839,7 @@ int LookUpCLMInventory(char * varName, varNCData **myVarNCData, int *clmIDs)
 
   return 0;
 #else
-  return 0;  
+  return 0;
 #endif
 }
 
@@ -883,7 +906,7 @@ void PutCLMDataInNC(int varID, Vector *v, double t, varNCData *myVarNCData, int
       int status = nc_put_vara_double(clmIDs[0], varID, start, count, &data_nc[0]);
       if (status != NC_NOERR)
       {
-	printf("Error: nc_put_vara_double failed, error code=%d\n", status);
+        printf("Error: nc_put_vara_double failed, error code=%d\n", status);
       }
       free(data_nc);
     }
@@ -932,7 +955,7 @@ void PutCLMDataInNC(int varID, Vector *v, double t, varNCData *myVarNCData, int
       int status = nc_put_vara_double(clmIDs[0], varID, start, count, &data_nc[0]);
       if (status != NC_NOERR)
       {
-	printf("Error: nc_put_vara_double failed, error code=%d\n", status);
+        printf("Error: nc_put_vara_double failed, error code=%d\n", status);
       }
       free(data_nc);
     }
diff --git a/pfsimulator/parflow_lib/write_parflow_binary.c b/pfsimulator/parflow_lib/write_parflow_binary.c
index 04454a49f..433a22f28 100644
--- a/pfsimulator/parflow_lib/write_parflow_binary.c
+++ b/pfsimulator/parflow_lib/write_parflow_binary.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routines to write a Vector to a file in full or scattered form.
@@ -52,6 +52,7 @@ long SizeofPFBinarySubvector(
   int nz_v = SubvectorNZ(subvector);
 
   int i, j, k, ai;
+
   PF_UNUSED(ai);
 
   long size;
diff --git a/pfsimulator/parflow_lib/write_parflow_netcdf.c b/pfsimulator/parflow_lib/write_parflow_netcdf.c
index 935a2c567..c547ed538 100644
--- a/pfsimulator/parflow_lib/write_parflow_netcdf.c
+++ b/pfsimulator/parflow_lib/write_parflow_netcdf.c
@@ -1,24 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  This file is part of Parflow. For details, see
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routines to write a Vector to a file in full or scattered form.
@@ -43,9 +49,9 @@ static bool isTdefined = false;
 
 void FreeVarNCData(varNCData* myVarNCData)
 {
-  if(myVarNCData)
+  if (myVarNCData)
   {
-    free(myVarNCData -> dimIDs);
+    free(myVarNCData->dimIDs);
     free(myVarNCData);
   }
 }
@@ -58,6 +64,7 @@ void WritePFNC(char * file_prefix, char* file_postfix, double t, Vector  *v, int
   char *switch_name;
   char key[IDB_MAX_KEY_LEN];
   static int netCDFIDs[5]; /* Here we store file and dimension IDs */
+  BeginTiming(NetcdfTimingIndex);
   sprintf(key, "NetCDF.NodeLevelIO");
   switch_name = GetStringDefault(key, "False");
   if (strcmp(switch_name, default_val) != 0)
@@ -152,7 +159,7 @@ void WritePFNC(char * file_prefix, char* file_postfix, double t, Vector  *v, int
       int userSpecSteps = GetInt("NetCDF.NumStepsPerFile");
       static char file_name[255];
 
-      varNCData *myVarNCData=NULL;
+      varNCData *myVarNCData = NULL;
 
       if (numStepsInFile == userSpecSteps * numVarTimeVariant)
       {
@@ -265,6 +272,7 @@ void WritePFNC(char * file_prefix, char* file_postfix, double t, Vector  *v, int
     }
     FreeVarNCData(myVarNCData);
   }
+  EndTiming(NetcdfTimingIndex);
 #else
   amps_Printf("Parflow not compiled with NetCDF, can't create NetCDF file\n");
 #endif
@@ -352,7 +360,6 @@ void CreateNCFileNode(char *file_name, Vector *v, int *netCDFIDs)
     {
       printf("Error: nc_create_par failed for file <%s>, error code=%d\n", file_name, res);
     }
-
   }
   else
   {
@@ -361,7 +368,6 @@ void CreateNCFileNode(char *file_name, Vector *v, int *netCDFIDs)
     {
       printf("Error: nc_create_par failed for file <%s>, error code=%d\n", file_name, res);
     }
-
   }
   nc_def_dim(netCDFIDs[0], "x", nX, &netCDFIDs[4]);
   nc_def_dim(netCDFIDs[0], "y", nY, &netCDFIDs[3]);
@@ -447,10 +453,10 @@ int LookUpInventory(char * varName, varNCData **myVarNCData, int *netCDFIDs)
         chunksize[3] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(netCDFIDs[0], pressVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(netCDFIDs[0],pressVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(netCDFIDs[0], pressVarID, 0, 1, compression_level);
       }
-
     }
     if (res == NC_ENAMEINUSE)
     {
@@ -489,8 +495,9 @@ int LookUpInventory(char * varName, varNCData **myVarNCData, int *netCDFIDs)
         chunksize[3] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(netCDFIDs[0], satVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(netCDFIDs[0],satVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(netCDFIDs[0], satVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -530,8 +537,9 @@ int LookUpInventory(char * varName, varNCData **myVarNCData, int *netCDFIDs)
         chunksize[3] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(netCDFIDs[0], maskVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(netCDFIDs[0],maskVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(netCDFIDs[0], maskVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -569,8 +577,9 @@ int LookUpInventory(char * varName, varNCData **myVarNCData, int *netCDFIDs)
         chunksize[2] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(netCDFIDs[0], manningsVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(netCDFIDs[0],manningsVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(netCDFIDs[0], manningsVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -610,8 +619,9 @@ int LookUpInventory(char * varName, varNCData **myVarNCData, int *netCDFIDs)
         chunksize[3] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(netCDFIDs[0], perm_xVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(netCDFIDs[0],perm_xVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(netCDFIDs[0], perm_xVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -651,8 +661,9 @@ int LookUpInventory(char * varName, varNCData **myVarNCData, int *netCDFIDs)
         chunksize[3] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(netCDFIDs[0], perm_yVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(netCDFIDs[0],perm_yVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(netCDFIDs[0], perm_yVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -692,8 +703,9 @@ int LookUpInventory(char * varName, varNCData **myVarNCData, int *netCDFIDs)
         chunksize[3] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(netCDFIDs[0], perm_zVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(netCDFIDs[0],perm_zVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(netCDFIDs[0], perm_zVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -733,8 +745,9 @@ int LookUpInventory(char * varName, varNCData **myVarNCData, int *netCDFIDs)
         chunksize[3] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(netCDFIDs[0], porosityVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(netCDFIDs[0],porosityVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(netCDFIDs[0], porosityVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -774,10 +787,10 @@ int LookUpInventory(char * varName, varNCData **myVarNCData, int *netCDFIDs)
         chunksize[3] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(netCDFIDs[0], specStorageVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(netCDFIDs[0],specStorageVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(netCDFIDs[0], specStorageVarID, 0, 1, compression_level);
       }
-
     }
     if (res == NC_ENAMEINUSE)
     {
@@ -814,8 +827,9 @@ int LookUpInventory(char * varName, varNCData **myVarNCData, int *netCDFIDs)
         chunksize[2] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(netCDFIDs[0], slopexVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(netCDFIDs[0],slopexVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(netCDFIDs[0], slopexVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -852,8 +866,9 @@ int LookUpInventory(char * varName, varNCData **myVarNCData, int *netCDFIDs)
         chunksize[2] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(netCDFIDs[0], slopeyVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(netCDFIDs[0],slopeyVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(netCDFIDs[0], slopeyVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -892,8 +907,9 @@ int LookUpInventory(char * varName, varNCData **myVarNCData, int *netCDFIDs)
         chunksize[3] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(netCDFIDs[0], dzmultVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(netCDFIDs[0],dzmultVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(netCDFIDs[0], dzmultVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -933,10 +949,10 @@ int LookUpInventory(char * varName, varNCData **myVarNCData, int *netCDFIDs)
         chunksize[3] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(netCDFIDs[0], evaptransVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(netCDFIDs[0],evaptransVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(netCDFIDs[0], evaptransVarID, 0, 1, compression_level);
       }
-
     }
     if (res == NC_ENAMEINUSE)
     {
@@ -975,8 +991,9 @@ int LookUpInventory(char * varName, varNCData **myVarNCData, int *netCDFIDs)
         chunksize[3] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(netCDFIDs[0], evaptrans_sumVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(netCDFIDs[0],evaptrans_sumVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(netCDFIDs[0], evaptrans_sumVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -1013,8 +1030,9 @@ int LookUpInventory(char * varName, varNCData **myVarNCData, int *netCDFIDs)
         chunksize[2] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(netCDFIDs[0], overland_sumVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(netCDFIDs[0],overland_sumVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(netCDFIDs[0], overland_sumVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -1052,8 +1070,9 @@ int LookUpInventory(char * varName, varNCData **myVarNCData, int *netCDFIDs)
         chunksize[2] = GetInt("NetCDF.ChunkX");
         nc_def_var_chunking(netCDFIDs[0], overland_bc_fluxVarID, NC_CHUNKED, chunksize);
       }
-      if (enable_netcdf_compression) {
-        nc_def_var_deflate(netCDFIDs[0],overland_bc_fluxVarID,0,1,compression_level);
+      if (enable_netcdf_compression)
+      {
+        nc_def_var_deflate(netCDFIDs[0], overland_bc_fluxVarID, 0, 1, compression_level);
       }
     }
     if (res == NC_ENAMEINUSE)
@@ -1131,7 +1150,7 @@ void PutDataInNC(int varID, Vector *v, double t, varNCData *myVarNCData, int dim
       int status = nc_put_vara_double(netCDFIDs[0], varID, start, count, &data_nc[0]);
       if (status != NC_NOERR)
       {
-	printf("Error: nc_put_vara_double failed, error code=%d\n", status);
+        printf("Error: nc_put_vara_double failed, error code=%d\n", status);
       }
       free(data_nc);
     }
@@ -1179,7 +1198,7 @@ void PutDataInNC(int varID, Vector *v, double t, varNCData *myVarNCData, int dim
       int status = nc_put_vara_double(netCDFIDs[0], varID, start, count, &data_nc[0]);
       if (status != NC_NOERR)
       {
-	printf("Error: nc_put_vara_double failed, error code=%d\n", status);
+        printf("Error: nc_put_vara_double failed, error code=%d\n", status);
       }
       free(data_nc);
     }
@@ -1203,7 +1222,6 @@ void PutDataInNCNode(int varID, double *data_nc_node, int *nodeXIndices, int *no
     {
       printf("Error: nc_put_vara_double failed, error code=%d\n", status);
     }
-
   }
   else
   {
@@ -1226,7 +1244,7 @@ void PutDataInNCNode(int varID, double *data_nc_node, int *nodeXIndices, int *no
       status = nc_put_vara_double(netCDFIDs[0], varID, start, count, &data_nc_node[index]);
       if (status != NC_NOERR)
       {
-	printf("Error: nc_put_vara_double failed, error code=%d\n", status);
+        printf("Error: nc_put_vara_double failed, error code=%d\n", status);
       }
     }
   }
diff --git a/pfsimulator/parflow_lib/write_parflow_silo.c b/pfsimulator/parflow_lib/write_parflow_silo.c
index dc3b508f4..489f56bbb 100644
--- a/pfsimulator/parflow_lib/write_parflow_silo.c
+++ b/pfsimulator/parflow_lib/write_parflow_silo.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routines to write a Vector to Silo file.
@@ -199,7 +199,7 @@ void     WriteSiloInit(char *file_prefix)
 {
 #ifdef HAVE_SILO
   char filename[2048];
-  
+
   int p = amps_Rank(amps_CommWorld);
   int P = amps_Size(amps_CommWorld);
 
@@ -268,8 +268,8 @@ void     WriteSiloInit(char *file_prefix)
                              "specific_storage",
                              "mask",
                              "dz_mult",
-			     "top_zindex",
-			     "top_patch",
+                             "top_zindex",
+                             "top_patch",
                              "eflx_lh_tot",
                              "eflx_lwrad_out",
                              "eflx_sh_tot",
@@ -289,9 +289,9 @@ void     WriteSiloInit(char *file_prefix)
                              "evaptranssum",
                              "overlandsum",
                              "overland_bc_flux",
-                             0};
+                             0 };
 
-    for(int i = 0; output_types[i]; i++)
+    for (int i = 0; output_types[i]; i++)
     {
       sprintf(filename, "%s/%s", file_prefix, output_types[i]);
       pf_mk_dir(filename);
@@ -322,13 +322,12 @@ void     WriteSilo(char *  file_prefix,
                    int     step,
                    char *  variable_name)
 {
-
 #ifdef HAVE_SILO
   Grid           *grid = VectorGrid(v);
   SubgridArray   *subgrids = GridSubgrids(grid);
   Subgrid        *subgrid;
   Subvector      *subvector;
-  
+
   int g;
   int p, P;
 
diff --git a/pfsimulator/parflow_lib/write_parflow_silo_pmpio.c b/pfsimulator/parflow_lib/write_parflow_silo_pmpio.c
index b54f32f9c..1fd4cbe42 100644
--- a/pfsimulator/parflow_lib/write_parflow_silo_pmpio.c
+++ b/pfsimulator/parflow_lib/write_parflow_silo_pmpio.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * Routines to write a Vector to Silo file.
@@ -101,7 +101,7 @@ void     WriteSiloPMPIOInit(char *file_prefix)
 {
 #if defined(HAVE_SILO) && defined(HAVE_MPI)
   char filename[2048];
-  
+
   int p = amps_Rank(amps_CommWorld);
   int P = amps_Size(amps_CommWorld);
 
@@ -116,7 +116,7 @@ void     WriteSiloPMPIOInit(char *file_prefix)
   if (strlen(compression_options))
   {
     DBSetCompression(compression_options);
-    if (db_errno  < 0)
+    if (db_errno < 0)
     {
       amps_Printf("Error: Compression options failed for SILO.CompressionOptions=%s\n", compression_options);
       amps_Printf("       This may mean SILO was not compiled with compression enabled\n");
@@ -198,7 +198,6 @@ void     WriteSiloPMPIO(char *  file_prefix,
                         int     step,
                         char *  variable_name)
 {
-
 #if defined(HAVE_SILO) && defined(HAVE_MPI)
   Grid           *grid = VectorGrid(v);
   SubgridArray   *subgrids = GridSubgrids(grid);
diff --git a/pfsimulator/third_party/cub/agent/agent_histogram.cuh b/pfsimulator/third_party/cub/agent/agent_histogram.cuh
deleted file mode 100644
index 37b1ec973..000000000
--- a/pfsimulator/third_party/cub/agent/agent_histogram.cuh
+++ /dev/null
@@ -1,787 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::AgentHistogram implements a stateful abstraction of CUDA thread blocks for participating in device-wide histogram .
- */
-
-#pragma once
-
-#include <iterator>
-
-#include "../util_type.cuh"
-#include "../block/block_load.cuh"
-#include "../grid/grid_queue.cuh"
-#include "../iterator/cache_modified_input_iterator.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/******************************************************************************
- * Tuning policy
- ******************************************************************************/
-
-/**
- *
- */
-enum BlockHistogramMemoryPreference
-{
-    GMEM,
-    SMEM,
-    BLEND
-};
-
-
-/**
- * Parameterizable tuning policy type for AgentHistogram
- */
-template <
-    int                             _BLOCK_THREADS,                 ///< Threads per thread block
-    int                             _PIXELS_PER_THREAD,             ///< Pixels per thread (per tile of input)
-    BlockLoadAlgorithm              _LOAD_ALGORITHM,                ///< The BlockLoad algorithm to use
-    CacheLoadModifier               _LOAD_MODIFIER,                 ///< Cache load modifier for reading input elements
-    bool                            _RLE_COMPRESS,                  ///< Whether to perform localized RLE to compress samples before histogramming
-    BlockHistogramMemoryPreference  _MEM_PREFERENCE,                ///< Whether to prefer privatized shared-memory bins (versus privatized global-memory bins)
-    bool                            _WORK_STEALING>                 ///< Whether to dequeue tiles from a global work queue
-struct AgentHistogramPolicy
-{
-    enum
-    {
-        BLOCK_THREADS           = _BLOCK_THREADS,                   ///< Threads per thread block
-        PIXELS_PER_THREAD       = _PIXELS_PER_THREAD,               ///< Pixels per thread (per tile of input)
-        IS_RLE_COMPRESS         = _RLE_COMPRESS,                    ///< Whether to perform localized RLE to compress samples before histogramming
-        MEM_PREFERENCE          = _MEM_PREFERENCE,                  ///< Whether to prefer privatized shared-memory bins (versus privatized global-memory bins)
-        IS_WORK_STEALING        = _WORK_STEALING,                   ///< Whether to dequeue tiles from a global work queue
-    };
-
-    static const BlockLoadAlgorithm     LOAD_ALGORITHM          = _LOAD_ALGORITHM;          ///< The BlockLoad algorithm to use
-    static const CacheLoadModifier      LOAD_MODIFIER           = _LOAD_MODIFIER;           ///< Cache load modifier for reading input elements
-};
-
-
-/******************************************************************************
- * Thread block abstractions
- ******************************************************************************/
-
-/**
- * \brief AgentHistogram implements a stateful abstraction of CUDA thread blocks for participating in device-wide histogram .
- */
-template <
-    typename    AgentHistogramPolicyT,     ///< Parameterized AgentHistogramPolicy tuning policy type
-    int         PRIVATIZED_SMEM_BINS,           ///< Number of privatized shared-memory histogram bins of any channel.  Zero indicates privatized counters to be maintained in device-accessible memory.
-    int         NUM_CHANNELS,                   ///< Number of channels interleaved in the input data.  Supports up to four channels.
-    int         NUM_ACTIVE_CHANNELS,            ///< Number of channels actively being histogrammed
-    typename    SampleIteratorT,                ///< Random-access input iterator type for reading samples
-    typename    CounterT,                       ///< Integer type for counting sample occurrences per histogram bin
-    typename    PrivatizedDecodeOpT,            ///< The transform operator type for determining privatized counter indices from samples, one for each channel
-    typename    OutputDecodeOpT,                ///< The transform operator type for determining output bin-ids from privatized counter indices, one for each channel
-    typename    OffsetT,                        ///< Signed integer type for global offsets
-    int         PTX_ARCH = CUB_PTX_ARCH>        ///< PTX compute capability
-struct AgentHistogram
-{
-    //---------------------------------------------------------------------
-    // Types and constants
-    //---------------------------------------------------------------------
-
-    /// The sample type of the input iterator
-    typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
-
-    /// The pixel type of SampleT
-    typedef typename CubVector<SampleT, NUM_CHANNELS>::Type PixelT;
-
-    /// The quad type of SampleT
-    typedef typename CubVector<SampleT, 4>::Type QuadT;
-
-    /// Constants
-    enum
-    {
-        BLOCK_THREADS           = AgentHistogramPolicyT::BLOCK_THREADS,
-
-        PIXELS_PER_THREAD       = AgentHistogramPolicyT::PIXELS_PER_THREAD,
-        SAMPLES_PER_THREAD      = PIXELS_PER_THREAD * NUM_CHANNELS,
-        QUADS_PER_THREAD        = SAMPLES_PER_THREAD / 4,
-
-        TILE_PIXELS             = PIXELS_PER_THREAD * BLOCK_THREADS,
-        TILE_SAMPLES            = SAMPLES_PER_THREAD * BLOCK_THREADS,
-
-        IS_RLE_COMPRESS            = AgentHistogramPolicyT::IS_RLE_COMPRESS,
-
-        MEM_PREFERENCE          = (PRIVATIZED_SMEM_BINS > 0) ?
-                                        AgentHistogramPolicyT::MEM_PREFERENCE :
-                                        GMEM,
-
-        IS_WORK_STEALING           = AgentHistogramPolicyT::IS_WORK_STEALING,
-    };
-
-    /// Cache load modifier for reading input elements
-    static const CacheLoadModifier LOAD_MODIFIER = AgentHistogramPolicyT::LOAD_MODIFIER;
-
-
-    /// Input iterator wrapper type (for applying cache modifier)
-    typedef typename If<IsPointer<SampleIteratorT>::VALUE,
-            CacheModifiedInputIterator<LOAD_MODIFIER, SampleT, OffsetT>,     // Wrap the native input pointer with CacheModifiedInputIterator
-            SampleIteratorT>::Type                                           // Directly use the supplied input iterator type
-        WrappedSampleIteratorT;
-
-    /// Pixel input iterator type (for applying cache modifier)
-    typedef CacheModifiedInputIterator<LOAD_MODIFIER, PixelT, OffsetT>
-        WrappedPixelIteratorT;
-
-    /// Qaud input iterator type (for applying cache modifier)
-    typedef CacheModifiedInputIterator<LOAD_MODIFIER, QuadT, OffsetT>
-        WrappedQuadIteratorT;
-
-    /// Parameterized BlockLoad type for samples
-    typedef BlockLoad<
-            SampleT,
-            BLOCK_THREADS,
-            SAMPLES_PER_THREAD,
-            AgentHistogramPolicyT::LOAD_ALGORITHM>
-        BlockLoadSampleT;
-
-    /// Parameterized BlockLoad type for pixels
-    typedef BlockLoad<
-            PixelT,
-            BLOCK_THREADS,
-            PIXELS_PER_THREAD,
-            AgentHistogramPolicyT::LOAD_ALGORITHM>
-        BlockLoadPixelT;
-
-    /// Parameterized BlockLoad type for quads
-    typedef BlockLoad<
-            QuadT,
-            BLOCK_THREADS,
-            QUADS_PER_THREAD,
-            AgentHistogramPolicyT::LOAD_ALGORITHM>
-        BlockLoadQuadT;
-
-    /// Shared memory type required by this thread block
-    struct _TempStorage
-    {
-        CounterT histograms[NUM_ACTIVE_CHANNELS][PRIVATIZED_SMEM_BINS + 1];     // Smem needed for block-privatized smem histogram (with 1 word of padding)
-
-        int tile_idx;
-
-        // Aliasable storage layout
-        union Aliasable
-        {
-            typename BlockLoadSampleT::TempStorage sample_load;     // Smem needed for loading a tile of samples
-            typename BlockLoadPixelT::TempStorage pixel_load;       // Smem needed for loading a tile of pixels
-            typename BlockLoadQuadT::TempStorage quad_load;         // Smem needed for loading a tile of quads
-
-        } aliasable;
-    };
-
-
-    /// Temporary storage type (unionable)
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    //---------------------------------------------------------------------
-    // Per-thread fields
-    //---------------------------------------------------------------------
-
-    /// Reference to temp_storage
-    _TempStorage &temp_storage;
-
-    /// Sample input iterator (with cache modifier applied, if possible)
-    WrappedSampleIteratorT d_wrapped_samples;
-
-    /// Native pointer for input samples (possibly NULL if unavailable)
-    SampleT* d_native_samples;
-
-    /// The number of output bins for each channel
-    int (&num_output_bins)[NUM_ACTIVE_CHANNELS];
-
-    /// The number of privatized bins for each channel
-    int (&num_privatized_bins)[NUM_ACTIVE_CHANNELS];
-
-    /// Reference to gmem privatized histograms for each channel
-    CounterT* d_privatized_histograms[NUM_ACTIVE_CHANNELS];
-
-    /// Reference to final output histograms (gmem)
-    CounterT* (&d_output_histograms)[NUM_ACTIVE_CHANNELS];
-
-    /// The transform operator for determining output bin-ids from privatized counter indices, one for each channel
-    OutputDecodeOpT (&output_decode_op)[NUM_ACTIVE_CHANNELS];
-
-    /// The transform operator for determining privatized counter indices from samples, one for each channel
-    PrivatizedDecodeOpT (&privatized_decode_op)[NUM_ACTIVE_CHANNELS];
-
-    /// Whether to prefer privatized smem counters vs privatized global counters
-    bool prefer_smem;
-
-
-    //---------------------------------------------------------------------
-    // Initialize privatized bin counters
-    //---------------------------------------------------------------------
-
-    // Initialize privatized bin counters
-    __device__ __forceinline__ void InitBinCounters(CounterT* privatized_histograms[NUM_ACTIVE_CHANNELS])
-    {
-        // Initialize histogram bin counts to zeros
-        #pragma unroll
-        for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
-        {
-            for (int privatized_bin = threadIdx.x; privatized_bin < num_privatized_bins[CHANNEL]; privatized_bin += BLOCK_THREADS)
-            {
-                privatized_histograms[CHANNEL][privatized_bin] = 0;
-            }
-        }
-
-        // Barrier to make sure all threads are done updating counters
-        CTA_SYNC();
-    }
-
-
-    // Initialize privatized bin counters.  Specialized for privatized shared-memory counters
-    __device__ __forceinline__ void InitSmemBinCounters()
-    {
-        CounterT* privatized_histograms[NUM_ACTIVE_CHANNELS];
-
-        for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
-            privatized_histograms[CHANNEL] = temp_storage.histograms[CHANNEL];
-
-        InitBinCounters(privatized_histograms);
-    }
-
-
-    // Initialize privatized bin counters.  Specialized for privatized global-memory counters
-    __device__ __forceinline__ void InitGmemBinCounters()
-    {
-        InitBinCounters(d_privatized_histograms);
-    }
-
-
-    //---------------------------------------------------------------------
-    // Update final output histograms
-    //---------------------------------------------------------------------
-
-    // Update final output histograms from privatized histograms
-    __device__ __forceinline__ void StoreOutput(CounterT* privatized_histograms[NUM_ACTIVE_CHANNELS])
-    {
-        // Barrier to make sure all threads are done updating counters
-        CTA_SYNC();
-
-        // Apply privatized bin counts to output bin counts
-        #pragma unroll
-        for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
-        {
-            int channel_bins = num_privatized_bins[CHANNEL];
-            for (int privatized_bin = threadIdx.x; 
-                    privatized_bin < channel_bins;  
-                    privatized_bin += BLOCK_THREADS)
-            {
-                int         output_bin  = -1;
-                CounterT    count       = privatized_histograms[CHANNEL][privatized_bin];
-                bool        is_valid    = count > 0;
-
-                output_decode_op[CHANNEL].template BinSelect<LOAD_MODIFIER>((SampleT) privatized_bin, output_bin, is_valid);
-
-                if (output_bin >= 0)
-                {
-                    atomicAdd(&d_output_histograms[CHANNEL][output_bin], count);
-                }
-
-            }
-        }
-    }
-
-
-    // Update final output histograms from privatized histograms.  Specialized for privatized shared-memory counters
-    __device__ __forceinline__ void StoreSmemOutput()
-    {
-        CounterT* privatized_histograms[NUM_ACTIVE_CHANNELS];
-        for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
-            privatized_histograms[CHANNEL] = temp_storage.histograms[CHANNEL];
-
-        StoreOutput(privatized_histograms);
-    }
-
-
-    // Update final output histograms from privatized histograms.  Specialized for privatized global-memory counters
-    __device__ __forceinline__ void StoreGmemOutput()
-    {
-        StoreOutput(d_privatized_histograms);
-    }
-
-
-    //---------------------------------------------------------------------
-    // Tile accumulation
-    //---------------------------------------------------------------------
-
-    // Accumulate pixels.  Specialized for RLE compression.
-    __device__ __forceinline__ void AccumulatePixels(
-        SampleT             samples[PIXELS_PER_THREAD][NUM_CHANNELS],
-        bool                is_valid[PIXELS_PER_THREAD],
-        CounterT*           privatized_histograms[NUM_ACTIVE_CHANNELS],
-        Int2Type<true>      is_rle_compress)
-    {
-        #pragma unroll
-        for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
-        {
-            // Bin pixels
-            int bins[PIXELS_PER_THREAD];
-
-            #pragma unroll
-            for (int PIXEL = 0; PIXEL < PIXELS_PER_THREAD; ++PIXEL)
-            {
-                bins[PIXEL] = -1;
-                privatized_decode_op[CHANNEL].template BinSelect<LOAD_MODIFIER>(samples[PIXEL][CHANNEL], bins[PIXEL], is_valid[PIXEL]);
-            }
-
-            CounterT accumulator = 1;
-
-            #pragma unroll
-            for (int PIXEL = 0; PIXEL < PIXELS_PER_THREAD - 1; ++PIXEL)
-            {
-                if (bins[PIXEL] != bins[PIXEL + 1])
-                {
-                    if (bins[PIXEL] >= 0)
-                        atomicAdd(privatized_histograms[CHANNEL] + bins[PIXEL], accumulator);
-
-                     accumulator = 0;
-                }
-                accumulator++;
-            }
-
-            // Last pixel
-            if (bins[PIXELS_PER_THREAD - 1] >= 0)
-                atomicAdd(privatized_histograms[CHANNEL] + bins[PIXELS_PER_THREAD - 1], accumulator);
-        }
-    }
-
-
-    // Accumulate pixels.  Specialized for individual accumulation of each pixel.
-    __device__ __forceinline__ void AccumulatePixels(
-        SampleT             samples[PIXELS_PER_THREAD][NUM_CHANNELS],
-        bool                is_valid[PIXELS_PER_THREAD],
-        CounterT*           privatized_histograms[NUM_ACTIVE_CHANNELS],
-        Int2Type<false>     is_rle_compress)
-    {
-        #pragma unroll
-        for (int PIXEL = 0; PIXEL < PIXELS_PER_THREAD; ++PIXEL)
-        {
-            #pragma unroll
-            for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
-            {
-                int bin = -1;
-                privatized_decode_op[CHANNEL].template BinSelect<LOAD_MODIFIER>(samples[PIXEL][CHANNEL], bin, is_valid[PIXEL]);
-                if (bin >= 0)
-                    atomicAdd(privatized_histograms[CHANNEL] + bin, 1);
-            }
-        }
-    }
-
-
-    /**
-     * Accumulate pixel, specialized for smem privatized histogram
-     */
-    __device__ __forceinline__ void AccumulateSmemPixels(
-        SampleT             samples[PIXELS_PER_THREAD][NUM_CHANNELS],
-        bool                is_valid[PIXELS_PER_THREAD])
-    {
-        CounterT* privatized_histograms[NUM_ACTIVE_CHANNELS];
-
-        for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
-            privatized_histograms[CHANNEL] = temp_storage.histograms[CHANNEL];
-
-        AccumulatePixels(samples, is_valid, privatized_histograms, Int2Type<IS_RLE_COMPRESS>());
-    }
-
-
-    /**
-     * Accumulate pixel, specialized for gmem privatized histogram
-     */
-    __device__ __forceinline__ void AccumulateGmemPixels(
-        SampleT             samples[PIXELS_PER_THREAD][NUM_CHANNELS],
-        bool                is_valid[PIXELS_PER_THREAD])
-    {
-        AccumulatePixels(samples, is_valid, d_privatized_histograms, Int2Type<IS_RLE_COMPRESS>());
-    }
-
-
-
-    //---------------------------------------------------------------------
-    // Tile loading
-    //---------------------------------------------------------------------
-
-    // Load full, aligned tile using pixel iterator (multi-channel)
-    template <int _NUM_ACTIVE_CHANNELS>
-    __device__ __forceinline__ void LoadFullAlignedTile(
-        OffsetT                         block_offset,
-        int                             valid_samples,
-        SampleT                         (&samples)[PIXELS_PER_THREAD][NUM_CHANNELS],
-        Int2Type<_NUM_ACTIVE_CHANNELS>  num_active_channels)
-    {
-        typedef PixelT AliasedPixels[PIXELS_PER_THREAD];
-
-        WrappedPixelIteratorT d_wrapped_pixels((PixelT*) (d_native_samples + block_offset));
-
-        // Load using a wrapped pixel iterator
-        BlockLoadPixelT(temp_storage.aliasable.pixel_load).Load(
-            d_wrapped_pixels,
-            reinterpret_cast<AliasedPixels&>(samples));
-    }
-
-    // Load full, aligned tile using quad iterator (single-channel)
-    __device__ __forceinline__ void LoadFullAlignedTile(
-        OffsetT                         block_offset,
-        int                             valid_samples,
-        SampleT                         (&samples)[PIXELS_PER_THREAD][NUM_CHANNELS],
-        Int2Type<1>                     num_active_channels)
-    {
-        typedef QuadT AliasedQuads[QUADS_PER_THREAD];
-
-        WrappedQuadIteratorT d_wrapped_quads((QuadT*) (d_native_samples + block_offset));
-
-        // Load using a wrapped quad iterator
-        BlockLoadQuadT(temp_storage.aliasable.quad_load).Load(
-            d_wrapped_quads,
-            reinterpret_cast<AliasedQuads&>(samples));
-    }
-
-    // Load full, aligned tile
-    __device__ __forceinline__ void LoadTile(
-        OffsetT         block_offset,
-        int             valid_samples,
-        SampleT         (&samples)[PIXELS_PER_THREAD][NUM_CHANNELS],
-        Int2Type<true>  is_full_tile,
-        Int2Type<true>  is_aligned)
-    {
-        LoadFullAlignedTile(block_offset, valid_samples, samples, Int2Type<NUM_ACTIVE_CHANNELS>());
-    }
-
-    // Load full, mis-aligned tile using sample iterator
-    __device__ __forceinline__ void LoadTile(
-        OffsetT         block_offset,
-        int             valid_samples,
-        SampleT         (&samples)[PIXELS_PER_THREAD][NUM_CHANNELS],
-        Int2Type<true>  is_full_tile,
-        Int2Type<false> is_aligned)
-    {
-        typedef SampleT AliasedSamples[SAMPLES_PER_THREAD];
-
-        // Load using sample iterator
-        BlockLoadSampleT(temp_storage.aliasable.sample_load).Load(
-            d_wrapped_samples + block_offset,
-            reinterpret_cast<AliasedSamples&>(samples));
-    }
-
-    // Load partially-full, aligned tile using the pixel iterator
-    __device__ __forceinline__ void LoadTile(
-        OffsetT         block_offset,
-        int             valid_samples,
-        SampleT         (&samples)[PIXELS_PER_THREAD][NUM_CHANNELS],
-        Int2Type<false> is_full_tile,
-        Int2Type<true>  is_aligned)
-    {
-        typedef PixelT AliasedPixels[PIXELS_PER_THREAD];
-
-        WrappedPixelIteratorT d_wrapped_pixels((PixelT*) (d_native_samples + block_offset));
-
-        int valid_pixels = valid_samples / NUM_CHANNELS;
-
-        // Load using a wrapped pixel iterator
-        BlockLoadPixelT(temp_storage.aliasable.pixel_load).Load(
-            d_wrapped_pixels,
-            reinterpret_cast<AliasedPixels&>(samples),
-            valid_pixels);
-    }
-
-    // Load partially-full, mis-aligned tile using sample iterator
-    __device__ __forceinline__ void LoadTile(
-        OffsetT         block_offset,
-        int             valid_samples,
-        SampleT         (&samples)[PIXELS_PER_THREAD][NUM_CHANNELS],
-        Int2Type<false> is_full_tile,
-        Int2Type<false> is_aligned)
-    {
-        typedef SampleT AliasedSamples[SAMPLES_PER_THREAD];
-
-        BlockLoadSampleT(temp_storage.aliasable.sample_load).Load(
-            d_wrapped_samples + block_offset,
-            reinterpret_cast<AliasedSamples&>(samples),
-            valid_samples);
-    }
-
-
-    //---------------------------------------------------------------------
-    // Tile processing
-    //---------------------------------------------------------------------
-
-    // Consume a tile of data samples
-    template <
-        bool IS_ALIGNED,        // Whether the tile offset is aligned (quad-aligned for single-channel, pixel-aligned for multi-channel)
-        bool IS_FULL_TILE>      // Whether the tile is full
-    __device__ __forceinline__ void ConsumeTile(OffsetT block_offset, int valid_samples)
-    {
-        SampleT     samples[PIXELS_PER_THREAD][NUM_CHANNELS];
-        bool        is_valid[PIXELS_PER_THREAD];
-
-        // Load tile
-        LoadTile(
-            block_offset,
-            valid_samples,
-            samples,
-            Int2Type<IS_FULL_TILE>(),
-            Int2Type<IS_ALIGNED>());
-
-        // Set valid flags
-        #pragma unroll
-        for (int PIXEL = 0; PIXEL < PIXELS_PER_THREAD; ++PIXEL)
-            is_valid[PIXEL] = IS_FULL_TILE || (((threadIdx.x * PIXELS_PER_THREAD + PIXEL) * NUM_CHANNELS) < valid_samples);
-
-        // Accumulate samples
-#if CUB_PTX_ARCH >= 120
-        if (prefer_smem)
-            AccumulateSmemPixels(samples, is_valid);
-        else
-            AccumulateGmemPixels(samples, is_valid);
-#else
-        AccumulateGmemPixels(samples, is_valid);
-#endif
-
-    }
-
-
-    // Consume row tiles.  Specialized for work-stealing from queue
-    template <bool IS_ALIGNED>
-    __device__ __forceinline__ void ConsumeTiles(
-        OffsetT             num_row_pixels,             ///< The number of multi-channel pixels per row in the region of interest
-        OffsetT             num_rows,                   ///< The number of rows in the region of interest
-        OffsetT             row_stride_samples,         ///< The number of samples between starts of consecutive rows in the region of interest
-        int                 tiles_per_row,              ///< Number of image tiles per row
-        GridQueue<int>      tile_queue,
-        Int2Type<true>      is_work_stealing)
-    {
-
-        int         num_tiles                   = num_rows * tiles_per_row;
-        int         tile_idx                    = (blockIdx.y  * gridDim.x) + blockIdx.x;
-        OffsetT     num_even_share_tiles        = gridDim.x * gridDim.y;
-
-        while (tile_idx < num_tiles)
-        {
-            int     row             = tile_idx / tiles_per_row;
-            int     col             = tile_idx - (row * tiles_per_row);
-            OffsetT row_offset      = row * row_stride_samples;
-            OffsetT col_offset      = (col * TILE_SAMPLES);
-            OffsetT tile_offset     = row_offset + col_offset;
-
-            if (col == tiles_per_row - 1)
-            {
-                // Consume a partially-full tile at the end of the row
-                OffsetT num_remaining = (num_row_pixels * NUM_CHANNELS) - col_offset;
-                ConsumeTile<IS_ALIGNED, false>(tile_offset, num_remaining);
-            } 
-            else
-            {
-                // Consume full tile
-                ConsumeTile<IS_ALIGNED, true>(tile_offset, TILE_SAMPLES);
-            }
-
-            CTA_SYNC();
-
-            // Get next tile
-            if (threadIdx.x == 0)
-                temp_storage.tile_idx = tile_queue.Drain(1) + num_even_share_tiles;
-
-            CTA_SYNC();
-
-            tile_idx = temp_storage.tile_idx;
-        }
-    }
-
-
-    // Consume row tiles.  Specialized for even-share (striped across thread blocks)
-    template <bool IS_ALIGNED>
-    __device__ __forceinline__ void ConsumeTiles(
-        OffsetT             num_row_pixels,             ///< The number of multi-channel pixels per row in the region of interest
-        OffsetT             num_rows,                   ///< The number of rows in the region of interest
-        OffsetT             row_stride_samples,         ///< The number of samples between starts of consecutive rows in the region of interest
-        int                 tiles_per_row,              ///< Number of image tiles per row
-        GridQueue<int>      tile_queue,
-        Int2Type<false>     is_work_stealing)
-    {
-        for (int row = blockIdx.y; row < num_rows; row += gridDim.y)
-        {
-            OffsetT row_begin   = row * row_stride_samples;
-            OffsetT row_end     = row_begin + (num_row_pixels * NUM_CHANNELS);
-            OffsetT tile_offset = row_begin + (blockIdx.x * TILE_SAMPLES);
-
-            while (tile_offset < row_end)
-            {
-                OffsetT num_remaining = row_end - tile_offset;
-
-                if (num_remaining < TILE_SAMPLES)
-                {
-                    // Consume partial tile
-                    ConsumeTile<IS_ALIGNED, false>(tile_offset, num_remaining);
-                    break;
-                }
-
-                // Consume full tile
-                ConsumeTile<IS_ALIGNED, true>(tile_offset, TILE_SAMPLES);
-                tile_offset += gridDim.x * TILE_SAMPLES;
-            }
-        }
-    }
-
-
-    //---------------------------------------------------------------------
-    // Parameter extraction
-    //---------------------------------------------------------------------
-
-    // Return a native pixel pointer (specialized for CacheModifiedInputIterator types)
-    template <
-        CacheLoadModifier   _MODIFIER,
-        typename            _ValueT,
-        typename            _OffsetT>
-    __device__ __forceinline__ SampleT* NativePointer(CacheModifiedInputIterator<_MODIFIER, _ValueT, _OffsetT> itr)
-    {
-        return itr.ptr;
-    }
-
-    // Return a native pixel pointer (specialized for other types)
-    template <typename IteratorT>
-    __device__ __forceinline__ SampleT* NativePointer(IteratorT itr)
-    {
-        return NULL;
-    }
-
-
-
-    //---------------------------------------------------------------------
-    // Interface
-    //---------------------------------------------------------------------
-
-
-    /**
-     * Constructor
-     */
-    __device__ __forceinline__ AgentHistogram(
-        TempStorage         &temp_storage,                                      ///< Reference to temp_storage
-        SampleIteratorT     d_samples,                                          ///< Input data to reduce
-        int                 (&num_output_bins)[NUM_ACTIVE_CHANNELS],            ///< The number bins per final output histogram
-        int                 (&num_privatized_bins)[NUM_ACTIVE_CHANNELS],        ///< The number bins per privatized histogram
-        CounterT*           (&d_output_histograms)[NUM_ACTIVE_CHANNELS],        ///< Reference to final output histograms
-        CounterT*           (&d_privatized_histograms)[NUM_ACTIVE_CHANNELS],    ///< Reference to privatized histograms
-        OutputDecodeOpT     (&output_decode_op)[NUM_ACTIVE_CHANNELS],           ///< The transform operator for determining output bin-ids from privatized counter indices, one for each channel
-        PrivatizedDecodeOpT (&privatized_decode_op)[NUM_ACTIVE_CHANNELS])       ///< The transform operator for determining privatized counter indices from samples, one for each channel
-    :
-        temp_storage(temp_storage.Alias()),
-        d_wrapped_samples(d_samples),
-        num_output_bins(num_output_bins),
-        num_privatized_bins(num_privatized_bins),
-        d_output_histograms(d_output_histograms),
-        privatized_decode_op(privatized_decode_op),
-        output_decode_op(output_decode_op),
-        d_native_samples(NativePointer(d_wrapped_samples)),
-        prefer_smem((MEM_PREFERENCE == SMEM) ?
-            true :                              // prefer smem privatized histograms
-            (MEM_PREFERENCE == GMEM) ?
-                false :                         // prefer gmem privatized histograms
-                blockIdx.x & 1)                 // prefer blended privatized histograms
-    {
-        int blockId = (blockIdx.y * gridDim.x) + blockIdx.x;
-
-        // Initialize the locations of this block's privatized histograms
-        for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
-            this->d_privatized_histograms[CHANNEL] = d_privatized_histograms[CHANNEL] + (blockId * num_privatized_bins[CHANNEL]);
-    }
-
-
-    /**
-     * Consume image
-     */
-    __device__ __forceinline__ void ConsumeTiles(
-        OffsetT             num_row_pixels,             ///< The number of multi-channel pixels per row in the region of interest
-        OffsetT             num_rows,                   ///< The number of rows in the region of interest
-        OffsetT             row_stride_samples,         ///< The number of samples between starts of consecutive rows in the region of interest
-        int                 tiles_per_row,              ///< Number of image tiles per row
-        GridQueue<int>      tile_queue)                 ///< Queue descriptor for assigning tiles of work to thread blocks
-    {
-        // Check whether all row starting offsets are quad-aligned (in single-channel) or pixel-aligned (in multi-channel)
-        int     quad_mask           = AlignBytes<QuadT>::ALIGN_BYTES - 1;
-        int     pixel_mask          = AlignBytes<PixelT>::ALIGN_BYTES - 1;
-        size_t  row_bytes           = sizeof(SampleT) * row_stride_samples;
-
-        bool quad_aligned_rows      = (NUM_CHANNELS == 1) && (SAMPLES_PER_THREAD % 4 == 0) &&     // Single channel
-                                        ((size_t(d_native_samples) & quad_mask) == 0) &&        // ptr is quad-aligned
-                                        ((num_rows == 1) || ((row_bytes & quad_mask) == 0));    // number of row-samples is a multiple of the alignment of the quad
-
-        bool pixel_aligned_rows     = (NUM_CHANNELS > 1) &&                                     // Multi channel
-                                        ((size_t(d_native_samples) & pixel_mask) == 0) &&       // ptr is pixel-aligned
-                                        ((row_bytes & pixel_mask) == 0);                        // number of row-samples is a multiple of the alignment of the pixel
-
-        // Whether rows are aligned and can be vectorized
-        if ((d_native_samples != NULL) && (quad_aligned_rows || pixel_aligned_rows))
-            ConsumeTiles<true>(num_row_pixels, num_rows, row_stride_samples, tiles_per_row, tile_queue, Int2Type<IS_WORK_STEALING>());
-        else
-            ConsumeTiles<false>(num_row_pixels, num_rows, row_stride_samples, tiles_per_row, tile_queue, Int2Type<IS_WORK_STEALING>());
-    }
-
-
-    /**
-     * Initialize privatized bin counters.  Specialized for privatized shared-memory counters
-     */
-    __device__ __forceinline__ void InitBinCounters()
-    {
-        if (prefer_smem)
-            InitSmemBinCounters();
-        else
-            InitGmemBinCounters();
-    }
-
-
-    /**
-     * Store privatized histogram to device-accessible memory.  Specialized for privatized shared-memory counters
-     */
-    __device__ __forceinline__ void StoreOutput()
-    {
-        if (prefer_smem)
-            StoreSmemOutput();
-        else
-            StoreGmemOutput();
-    }
-
-
-};
-
-
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/agent/agent_radix_sort_downsweep.cuh b/pfsimulator/third_party/cub/agent/agent_radix_sort_downsweep.cuh
deleted file mode 100644
index faea88138..000000000
--- a/pfsimulator/third_party/cub/agent/agent_radix_sort_downsweep.cuh
+++ /dev/null
@@ -1,789 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * AgentRadixSortDownsweep implements a stateful abstraction of CUDA thread blocks for participating in device-wide radix sort downsweep .
- */
-
-
-#pragma once
-
-#include <stdint.h>
-
-#include "../thread/thread_load.cuh"
-#include "../block/block_load.cuh"
-#include "../block/block_store.cuh"
-#include "../block/block_radix_rank.cuh"
-#include "../block/block_exchange.cuh"
-#include "../util_type.cuh"
-#include "../iterator/cache_modified_input_iterator.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/******************************************************************************
- * Tuning policy types
- ******************************************************************************/
-
-/**
- * Radix ranking algorithm
- */
-enum RadixRankAlgorithm
-{
-    RADIX_RANK_BASIC,
-    RADIX_RANK_MEMOIZE,
-    RADIX_RANK_MATCH
-};
-
-/**
- * Parameterizable tuning policy type for AgentRadixSortDownsweep
- */
-template <
-    int                         _BLOCK_THREADS,         ///< Threads per thread block
-    int                         _ITEMS_PER_THREAD,      ///< Items per thread (per tile of input)
-    BlockLoadAlgorithm          _LOAD_ALGORITHM,        ///< The BlockLoad algorithm to use
-    CacheLoadModifier           _LOAD_MODIFIER,         ///< Cache load modifier for reading keys (and values)
-    RadixRankAlgorithm          _RANK_ALGORITHM,        ///< The radix ranking algorithm to use
-    BlockScanAlgorithm          _SCAN_ALGORITHM,        ///< The block scan algorithm to use
-    int                         _RADIX_BITS>            ///< The number of radix bits, i.e., log2(bins)
-struct AgentRadixSortDownsweepPolicy
-{
-    enum
-    {
-        BLOCK_THREADS           = _BLOCK_THREADS,           ///< Threads per thread block
-        ITEMS_PER_THREAD        = _ITEMS_PER_THREAD,        ///< Items per thread (per tile of input)
-        RADIX_BITS              = _RADIX_BITS,              ///< The number of radix bits, i.e., log2(bins)
-    };
-
-    static const BlockLoadAlgorithm  LOAD_ALGORITHM     = _LOAD_ALGORITHM;    ///< The BlockLoad algorithm to use
-    static const CacheLoadModifier   LOAD_MODIFIER      = _LOAD_MODIFIER;     ///< Cache load modifier for reading keys (and values)
-    static const RadixRankAlgorithm  RANK_ALGORITHM     = _RANK_ALGORITHM;    ///< The radix ranking algorithm to use
-    static const BlockScanAlgorithm  SCAN_ALGORITHM     = _SCAN_ALGORITHM;    ///< The BlockScan algorithm to use
-};
-
-
-/******************************************************************************
- * Thread block abstractions
- ******************************************************************************/
-
-
-
-
-
-/**
- * \brief AgentRadixSortDownsweep implements a stateful abstraction of CUDA thread blocks for participating in device-wide radix sort downsweep .
- */
-template <
-    typename AgentRadixSortDownsweepPolicy,     ///< Parameterized AgentRadixSortDownsweepPolicy tuning policy type
-    bool     IS_DESCENDING,                     ///< Whether or not the sorted-order is high-to-low
-    typename KeyT,                              ///< KeyT type
-    typename ValueT,                            ///< ValueT type
-    typename OffsetT>                           ///< Signed integer type for global offsets
-struct AgentRadixSortDownsweep
-{
-    //---------------------------------------------------------------------
-    // Type definitions and constants
-    //---------------------------------------------------------------------
-
-    // Appropriate unsigned-bits representation of KeyT
-    typedef typename Traits<KeyT>::UnsignedBits UnsignedBits;
-
-    static const UnsignedBits           LOWEST_KEY  = Traits<KeyT>::LOWEST_KEY;
-    static const UnsignedBits           MAX_KEY     = Traits<KeyT>::MAX_KEY;
-
-    static const BlockLoadAlgorithm     LOAD_ALGORITHM  = AgentRadixSortDownsweepPolicy::LOAD_ALGORITHM;
-    static const CacheLoadModifier      LOAD_MODIFIER   = AgentRadixSortDownsweepPolicy::LOAD_MODIFIER;
-    static const RadixRankAlgorithm     RANK_ALGORITHM  = AgentRadixSortDownsweepPolicy::RANK_ALGORITHM;
-    static const BlockScanAlgorithm     SCAN_ALGORITHM  = AgentRadixSortDownsweepPolicy::SCAN_ALGORITHM;
-
-    enum
-    {
-        BLOCK_THREADS           = AgentRadixSortDownsweepPolicy::BLOCK_THREADS,
-        ITEMS_PER_THREAD        = AgentRadixSortDownsweepPolicy::ITEMS_PER_THREAD,
-        RADIX_BITS              = AgentRadixSortDownsweepPolicy::RADIX_BITS,
-        TILE_ITEMS              = BLOCK_THREADS * ITEMS_PER_THREAD,
-
-        RADIX_DIGITS            = 1 << RADIX_BITS,
-        KEYS_ONLY               = Equals<ValueT, NullType>::VALUE,
-    };
-
-    // Input iterator wrapper type (for applying cache modifier)s
-    typedef CacheModifiedInputIterator<LOAD_MODIFIER, UnsignedBits, OffsetT>    KeysItr;
-    typedef CacheModifiedInputIterator<LOAD_MODIFIER, ValueT, OffsetT>          ValuesItr;
-
-    // Radix ranking type to use
-    typedef typename If<(RANK_ALGORITHM == RADIX_RANK_BASIC),
-            BlockRadixRank<BLOCK_THREADS, RADIX_BITS, IS_DESCENDING, false, SCAN_ALGORITHM>,
-            typename If<(RANK_ALGORITHM == RADIX_RANK_MEMOIZE),
-                BlockRadixRank<BLOCK_THREADS, RADIX_BITS, IS_DESCENDING, true, SCAN_ALGORITHM>,
-                BlockRadixRankMatch<BLOCK_THREADS, RADIX_BITS, IS_DESCENDING, SCAN_ALGORITHM>
-            >::Type
-        >::Type BlockRadixRankT;
-
-    enum
-    {
-        /// Number of bin-starting offsets tracked per thread
-        BINS_TRACKED_PER_THREAD = BlockRadixRankT::BINS_TRACKED_PER_THREAD
-    };
-
-    // BlockLoad type (keys)
-    typedef BlockLoad<
-        UnsignedBits,
-        BLOCK_THREADS,
-        ITEMS_PER_THREAD,
-        LOAD_ALGORITHM> BlockLoadKeysT;
-
-    // BlockLoad type (values)
-    typedef BlockLoad<
-        ValueT,
-        BLOCK_THREADS,
-        ITEMS_PER_THREAD,
-        LOAD_ALGORITHM> BlockLoadValuesT;
-
-    // Value exchange array type
-    typedef ValueT ValueExchangeT[TILE_ITEMS];
-
-    /**
-     * Shared memory storage layout
-     */
-    union __align__(16) _TempStorage
-    {
-        typename BlockLoadKeysT::TempStorage    load_keys;
-        typename BlockLoadValuesT::TempStorage  load_values;
-        typename BlockRadixRankT::TempStorage   radix_rank;
-
-        struct
-        {
-            UnsignedBits                        exchange_keys[TILE_ITEMS];
-            OffsetT                             relative_bin_offsets[RADIX_DIGITS];
-        };
-
-        Uninitialized<ValueExchangeT>           exchange_values;
-
-        OffsetT                                 exclusive_digit_prefix[RADIX_DIGITS];
-    };
-
-
-    /// Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    //---------------------------------------------------------------------
-    // Thread fields
-    //---------------------------------------------------------------------
-
-    // Shared storage for this CTA
-    _TempStorage    &temp_storage;
-
-    // Input and output device pointers
-    KeysItr         d_keys_in;
-    ValuesItr       d_values_in;
-    UnsignedBits    *d_keys_out;
-    ValueT          *d_values_out;
-
-    // The global scatter base offset for each digit (valid in the first RADIX_DIGITS threads)
-    OffsetT         bin_offset[BINS_TRACKED_PER_THREAD];
-
-    // The least-significant bit position of the current digit to extract
-    int             current_bit;
-
-    // Number of bits in current digit
-    int             num_bits;
-
-    // Whether to short-cirucit
-    int             short_circuit;
-
-    //---------------------------------------------------------------------
-    // Utility methods
-    //---------------------------------------------------------------------
-
-
-    /**
-     * Scatter ranked keys through shared memory, then to device-accessible memory
-     */
-    template <bool FULL_TILE>
-    __device__ __forceinline__ void ScatterKeys(
-        UnsignedBits    (&twiddled_keys)[ITEMS_PER_THREAD],
-        OffsetT         (&relative_bin_offsets)[ITEMS_PER_THREAD],
-        int             (&ranks)[ITEMS_PER_THREAD],
-        OffsetT         valid_items)
-    {
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            temp_storage.exchange_keys[ranks[ITEM]] = twiddled_keys[ITEM];
-        }
-
-        CTA_SYNC();
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            UnsignedBits key            = temp_storage.exchange_keys[threadIdx.x + (ITEM * BLOCK_THREADS)];
-            UnsignedBits digit          = BFE(key, current_bit, num_bits);
-            relative_bin_offsets[ITEM]  = temp_storage.relative_bin_offsets[digit];
-
-            // Un-twiddle
-            key = Traits<KeyT>::TwiddleOut(key);
-
-            if (FULL_TILE || 
-                (static_cast<OffsetT>(threadIdx.x + (ITEM * BLOCK_THREADS)) < valid_items))
-            {
-                d_keys_out[relative_bin_offsets[ITEM] + threadIdx.x + (ITEM * BLOCK_THREADS)] = key;
-            }
-        }
-    }
-
-
-    /**
-     * Scatter ranked values through shared memory, then to device-accessible memory
-     */
-    template <bool FULL_TILE>
-    __device__ __forceinline__ void ScatterValues(
-        ValueT      (&values)[ITEMS_PER_THREAD],
-        OffsetT     (&relative_bin_offsets)[ITEMS_PER_THREAD],
-        int         (&ranks)[ITEMS_PER_THREAD],
-        OffsetT     valid_items)
-    {
-        CTA_SYNC();
-
-        ValueExchangeT &exchange_values = temp_storage.exchange_values.Alias();
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            exchange_values[ranks[ITEM]] = values[ITEM];
-        }
-
-        CTA_SYNC();
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            ValueT value = exchange_values[threadIdx.x + (ITEM * BLOCK_THREADS)];
-
-            if (FULL_TILE ||
-                (static_cast<OffsetT>(threadIdx.x + (ITEM * BLOCK_THREADS)) < valid_items))
-            {
-                d_values_out[relative_bin_offsets[ITEM] + threadIdx.x + (ITEM * BLOCK_THREADS)] = value;
-            }
-        }
-    }
-
-    /**
-     * Load a tile of keys (specialized for full tile, any ranking algorithm)
-     */
-    template <int _RANK_ALGORITHM>
-    __device__ __forceinline__ void LoadKeys(
-        UnsignedBits                (&keys)[ITEMS_PER_THREAD],
-        OffsetT                     block_offset,
-        OffsetT                     valid_items,
-        UnsignedBits                oob_item,
-        Int2Type<true>              is_full_tile,
-        Int2Type<_RANK_ALGORITHM>   rank_algorithm)
-    {
-        BlockLoadKeysT(temp_storage.load_keys).Load(
-            d_keys_in + block_offset, keys);
-
-        CTA_SYNC();
-    }
-
-
-    /**
-     * Load a tile of keys (specialized for partial tile, any ranking algorithm)
-     */
-    template <int _RANK_ALGORITHM>
-    __device__ __forceinline__ void LoadKeys(
-        UnsignedBits                (&keys)[ITEMS_PER_THREAD],
-        OffsetT                     block_offset,
-        OffsetT                     valid_items,
-        UnsignedBits                oob_item,
-        Int2Type<false>             is_full_tile,
-        Int2Type<_RANK_ALGORITHM>   rank_algorithm)
-    {
-        // Register pressure work-around: moving valid_items through shfl prevents compiler
-        // from reusing guards/addressing from prior guarded loads
-        valid_items = ShuffleIndex<CUB_PTX_WARP_THREADS>(valid_items, 0, 0xffffffff);
-
-        BlockLoadKeysT(temp_storage.load_keys).Load(
-            d_keys_in + block_offset, keys, valid_items, oob_item);
-
-        CTA_SYNC();
-    }
-
-
-    /**
-     * Load a tile of keys (specialized for full tile, match ranking algorithm)
-     */
-    __device__ __forceinline__ void LoadKeys(
-        UnsignedBits                (&keys)[ITEMS_PER_THREAD],
-        OffsetT                     block_offset,
-        OffsetT                     valid_items,
-        UnsignedBits                oob_item,
-        Int2Type<true>              is_full_tile,
-        Int2Type<RADIX_RANK_MATCH>  rank_algorithm)
-    {
-        LoadDirectWarpStriped(threadIdx.x, d_keys_in + block_offset, keys);
-    }
-
-
-    /**
-     * Load a tile of keys (specialized for partial tile, match ranking algorithm)
-     */
-    __device__ __forceinline__ void LoadKeys(
-        UnsignedBits                (&keys)[ITEMS_PER_THREAD],
-        OffsetT                     block_offset,
-        OffsetT                     valid_items,
-        UnsignedBits                oob_item,
-        Int2Type<false>             is_full_tile,
-        Int2Type<RADIX_RANK_MATCH>  rank_algorithm)
-    {
-        // Register pressure work-around: moving valid_items through shfl prevents compiler
-        // from reusing guards/addressing from prior guarded loads
-        valid_items = ShuffleIndex<CUB_PTX_WARP_THREADS>(valid_items, 0, 0xffffffff);
-
-        LoadDirectWarpStriped(threadIdx.x, d_keys_in + block_offset, keys, valid_items, oob_item);
-    }
-
-
-    /**
-     * Load a tile of values (specialized for full tile, any ranking algorithm)
-     */
-    template <int _RANK_ALGORITHM>
-    __device__ __forceinline__ void LoadValues(
-        ValueT                      (&values)[ITEMS_PER_THREAD],
-        OffsetT                     block_offset,
-        OffsetT                     valid_items,
-        Int2Type<true>              is_full_tile,
-        Int2Type<_RANK_ALGORITHM>   rank_algorithm)
-    {
-        BlockLoadValuesT(temp_storage.load_values).Load(
-            d_values_in + block_offset, values);
-
-        CTA_SYNC();
-    }
-
-
-    /**
-     * Load a tile of values (specialized for partial tile, any ranking algorithm)
-     */
-    template <int _RANK_ALGORITHM>
-    __device__ __forceinline__ void LoadValues(
-        ValueT                      (&values)[ITEMS_PER_THREAD],
-        OffsetT                     block_offset,
-        OffsetT                     valid_items,
-        Int2Type<false>             is_full_tile,
-        Int2Type<_RANK_ALGORITHM>   rank_algorithm)
-    {
-        // Register pressure work-around: moving valid_items through shfl prevents compiler
-        // from reusing guards/addressing from prior guarded loads
-        valid_items = ShuffleIndex<CUB_PTX_WARP_THREADS>(valid_items, 0, 0xffffffff);
-
-        BlockLoadValuesT(temp_storage.load_values).Load(
-            d_values_in + block_offset, values, valid_items);
-
-        CTA_SYNC();
-    }
-
-
-    /**
-     * Load a tile of items (specialized for full tile, match ranking algorithm)
-     */
-    __device__ __forceinline__ void LoadValues(
-        ValueT                      (&values)[ITEMS_PER_THREAD],
-        OffsetT                     block_offset,
-        OffsetT                     valid_items,
-        Int2Type<true>              is_full_tile,
-        Int2Type<RADIX_RANK_MATCH>  rank_algorithm)
-    {
-        LoadDirectWarpStriped(threadIdx.x, d_values_in + block_offset, values);
-    }
-
-
-    /**
-     * Load a tile of items (specialized for partial tile, match ranking algorithm)
-     */
-    __device__ __forceinline__ void LoadValues(
-        ValueT                      (&values)[ITEMS_PER_THREAD],
-        OffsetT                     block_offset,
-        OffsetT                     valid_items,
-        Int2Type<false>             is_full_tile,
-        Int2Type<RADIX_RANK_MATCH>  rank_algorithm)
-    {
-        // Register pressure work-around: moving valid_items through shfl prevents compiler
-        // from reusing guards/addressing from prior guarded loads
-        valid_items = ShuffleIndex<CUB_PTX_WARP_THREADS>(valid_items, 0, 0xffffffff);
-
-        LoadDirectWarpStriped(threadIdx.x, d_values_in + block_offset, values, valid_items);
-    }
-
-
-    /**
-     * Truck along associated values
-     */
-    template <bool FULL_TILE>
-    __device__ __forceinline__ void GatherScatterValues(
-        OffsetT         (&relative_bin_offsets)[ITEMS_PER_THREAD],
-        int             (&ranks)[ITEMS_PER_THREAD],
-        OffsetT         block_offset,
-        OffsetT         valid_items,
-        Int2Type<false> /*is_keys_only*/)
-    {
-        ValueT values[ITEMS_PER_THREAD];
-
-        CTA_SYNC();
-
-        LoadValues(
-            values,
-            block_offset,
-            valid_items,
-            Int2Type<FULL_TILE>(),
-            Int2Type<RANK_ALGORITHM>());
-
-        ScatterValues<FULL_TILE>(
-            values,
-            relative_bin_offsets,
-            ranks,
-            valid_items);
-    }
-
-
-    /**
-     * Truck along associated values (specialized for key-only sorting)
-     */
-    template <bool FULL_TILE>
-    __device__ __forceinline__ void GatherScatterValues(
-        OffsetT         (&/*relative_bin_offsets*/)[ITEMS_PER_THREAD],
-        int             (&/*ranks*/)[ITEMS_PER_THREAD],
-        OffsetT         /*block_offset*/,
-        OffsetT         /*valid_items*/,
-        Int2Type<true>  /*is_keys_only*/)
-    {}
-
-
-    /**
-     * Process tile
-     */
-    template <bool FULL_TILE>
-    __device__ __forceinline__ void ProcessTile(
-        OffsetT block_offset,
-        const OffsetT &valid_items = TILE_ITEMS)
-    {
-        UnsignedBits    keys[ITEMS_PER_THREAD];
-        int             ranks[ITEMS_PER_THREAD];
-        OffsetT         relative_bin_offsets[ITEMS_PER_THREAD];
-
-        // Assign default (min/max) value to all keys
-        UnsignedBits default_key = (IS_DESCENDING) ? LOWEST_KEY : MAX_KEY;
-
-        // Load tile of keys
-        LoadKeys(
-            keys,
-            block_offset,
-            valid_items, 
-            default_key,
-            Int2Type<FULL_TILE>(),
-            Int2Type<RANK_ALGORITHM>());
-
-        // Twiddle key bits if necessary
-        #pragma unroll
-        for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++)
-        {
-            keys[KEY] = Traits<KeyT>::TwiddleIn(keys[KEY]);
-        }
-
-        // Rank the twiddled keys
-        int exclusive_digit_prefix[BINS_TRACKED_PER_THREAD];
-        BlockRadixRankT(temp_storage.radix_rank).RankKeys(
-            keys,
-            ranks,
-            current_bit,
-            num_bits,
-            exclusive_digit_prefix);
-
-        CTA_SYNC();
-
-        // Share exclusive digit prefix
-        #pragma unroll
-        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
-        {
-            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
-            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
-            {
-                // Store exclusive prefix
-                temp_storage.exclusive_digit_prefix[bin_idx] =
-                    exclusive_digit_prefix[track];
-            }
-        }
-
-        CTA_SYNC();
-
-        // Get inclusive digit prefix
-        int inclusive_digit_prefix[BINS_TRACKED_PER_THREAD];
-
-        #pragma unroll
-        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
-        {
-            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
-            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
-            {
-                if (IS_DESCENDING)
-                {
-                    // Get inclusive digit prefix from exclusive prefix (higher bins come first)
-                    inclusive_digit_prefix[track] = (bin_idx == 0) ?
-                        (BLOCK_THREADS * ITEMS_PER_THREAD) :
-                        temp_storage.exclusive_digit_prefix[bin_idx - 1];
-                }
-                else
-                {
-                    // Get inclusive digit prefix from exclusive prefix (lower bins come first)
-                    inclusive_digit_prefix[track] = (bin_idx == RADIX_DIGITS - 1) ?
-                        (BLOCK_THREADS * ITEMS_PER_THREAD) :
-                        temp_storage.exclusive_digit_prefix[bin_idx + 1];
-                }
-            }
-        }
-
-        CTA_SYNC();
-
-        // Update global scatter base offsets for each digit
-        #pragma unroll
-        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
-        {
-            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
-            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
-            {
-                bin_offset[track] -= exclusive_digit_prefix[track];
-                temp_storage.relative_bin_offsets[bin_idx] = bin_offset[track];
-                bin_offset[track] += inclusive_digit_prefix[track];
-            }
-        }
-
-        CTA_SYNC();
-
-        // Scatter keys
-        ScatterKeys<FULL_TILE>(keys, relative_bin_offsets, ranks, valid_items);
-
-        // Gather/scatter values
-        GatherScatterValues<FULL_TILE>(relative_bin_offsets , ranks, block_offset, valid_items, Int2Type<KEYS_ONLY>());
-    }
-
-    //---------------------------------------------------------------------
-    // Copy shortcut
-    //---------------------------------------------------------------------
-
-    /**
-     * Copy tiles within the range of input
-     */
-    template <
-        typename InputIteratorT,
-        typename T>
-    __device__ __forceinline__ void Copy(
-        InputIteratorT  d_in,
-        T               *d_out,
-        OffsetT         block_offset,
-        OffsetT         block_end)
-    {
-        // Simply copy the input
-        while (block_offset + TILE_ITEMS <= block_end)
-        {
-            T items[ITEMS_PER_THREAD];
-
-            LoadDirectStriped<BLOCK_THREADS>(threadIdx.x, d_in + block_offset, items);
-            CTA_SYNC();
-            StoreDirectStriped<BLOCK_THREADS>(threadIdx.x, d_out + block_offset, items);
-
-            block_offset += TILE_ITEMS;
-        }
-
-        // Clean up last partial tile with guarded-I/O
-        if (block_offset < block_end)
-        {
-            OffsetT valid_items = block_end - block_offset;
-
-            T items[ITEMS_PER_THREAD];
-
-            LoadDirectStriped<BLOCK_THREADS>(threadIdx.x, d_in + block_offset, items, valid_items);
-            CTA_SYNC();
-            StoreDirectStriped<BLOCK_THREADS>(threadIdx.x, d_out + block_offset, items, valid_items);
-        }
-    }
-
-
-    /**
-     * Copy tiles within the range of input (specialized for NullType)
-     */
-    template <typename InputIteratorT>
-    __device__ __forceinline__ void Copy(
-        InputIteratorT  /*d_in*/,
-        NullType        * /*d_out*/,
-        OffsetT         /*block_offset*/,
-        OffsetT         /*block_end*/)
-    {}
-
-
-    //---------------------------------------------------------------------
-    // Interface
-    //---------------------------------------------------------------------
-
-    /**
-     * Constructor
-     */
-    __device__ __forceinline__ AgentRadixSortDownsweep(
-        TempStorage     &temp_storage,
-        OffsetT         (&bin_offset)[BINS_TRACKED_PER_THREAD],
-        OffsetT         num_items,
-        const KeyT      *d_keys_in,
-        KeyT            *d_keys_out,
-        const ValueT    *d_values_in,
-        ValueT          *d_values_out,
-        int             current_bit,
-        int             num_bits)
-    :
-        temp_storage(temp_storage.Alias()),
-        d_keys_in(reinterpret_cast<const UnsignedBits*>(d_keys_in)),
-        d_values_in(d_values_in),
-        d_keys_out(reinterpret_cast<UnsignedBits*>(d_keys_out)),
-        d_values_out(d_values_out),
-        current_bit(current_bit),
-        num_bits(num_bits),
-        short_circuit(1)
-    {
-        #pragma unroll
-        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
-        {
-            this->bin_offset[track] = bin_offset[track];
-
-            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
-            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
-            {
-                // Short circuit if the histogram has only bin counts of only zeros or problem-size
-                short_circuit = short_circuit && ((bin_offset[track] == 0) || (bin_offset[track] == num_items));
-            }
-        }
-
-        short_circuit = CTA_SYNC_AND(short_circuit);
-    }
-
-
-    /**
-     * Constructor
-     */
-    __device__ __forceinline__ AgentRadixSortDownsweep(
-        TempStorage     &temp_storage,
-        OffsetT         num_items,
-        OffsetT         *d_spine,
-        const KeyT      *d_keys_in,
-        KeyT            *d_keys_out,
-        const ValueT    *d_values_in,
-        ValueT          *d_values_out,
-        int             current_bit,
-        int             num_bits)
-    :
-        temp_storage(temp_storage.Alias()),
-        d_keys_in(reinterpret_cast<const UnsignedBits*>(d_keys_in)),
-        d_values_in(d_values_in),
-        d_keys_out(reinterpret_cast<UnsignedBits*>(d_keys_out)),
-        d_values_out(d_values_out),
-        current_bit(current_bit),
-        num_bits(num_bits),
-        short_circuit(1)
-    {
-        #pragma unroll
-        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
-        {
-            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
-
-            // Load digit bin offsets (each of the first RADIX_DIGITS threads will load an offset for that digit)
-            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
-            {
-                if (IS_DESCENDING)
-                    bin_idx = RADIX_DIGITS - bin_idx - 1;
-
-                // Short circuit if the first block's histogram has only bin counts of only zeros or problem-size
-                OffsetT first_block_bin_offset = d_spine[gridDim.x * bin_idx];
-                short_circuit = short_circuit && ((first_block_bin_offset == 0) || (first_block_bin_offset == num_items));
-
-                // Load my block's bin offset for my bin
-                bin_offset[track] = d_spine[(gridDim.x * bin_idx) + blockIdx.x];
-            }
-        }
-
-        short_circuit = CTA_SYNC_AND(short_circuit);
-    }
-
-
-    /**
-     * Distribute keys from a segment of input tiles.
-     */
-    __device__ __forceinline__ void ProcessRegion(
-        OffsetT   block_offset,
-        OffsetT   block_end)
-    {
-        if (short_circuit)
-        {
-            // Copy keys
-            Copy(d_keys_in, d_keys_out, block_offset, block_end);
-
-            // Copy values
-            Copy(d_values_in, d_values_out, block_offset, block_end);
-        }
-        else
-        {
-            // Process full tiles of tile_items
-            #pragma unroll 1
-            while (block_offset + TILE_ITEMS <= block_end)
-            {
-                ProcessTile<true>(block_offset);
-                block_offset += TILE_ITEMS;
-
-                CTA_SYNC();
-            }
-
-            // Clean up last partial tile with guarded-I/O
-            if (block_offset < block_end)
-            {
-                ProcessTile<false>(block_offset, block_end - block_offset);
-            }
-
-        }
-    }
-
-};
-
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/agent/agent_radix_sort_upsweep.cuh b/pfsimulator/third_party/cub/agent/agent_radix_sort_upsweep.cuh
deleted file mode 100644
index 2081cefba..000000000
--- a/pfsimulator/third_party/cub/agent/agent_radix_sort_upsweep.cuh
+++ /dev/null
@@ -1,526 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * AgentRadixSortUpsweep implements a stateful abstraction of CUDA thread blocks for participating in device-wide radix sort upsweep .
- */
-
-#pragma once
-
-#include "../thread/thread_reduce.cuh"
-#include "../thread/thread_load.cuh"
-#include "../warp/warp_reduce.cuh"
-#include "../block/block_load.cuh"
-#include "../util_type.cuh"
-#include "../iterator/cache_modified_input_iterator.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/******************************************************************************
- * Tuning policy types
- ******************************************************************************/
-
-/**
- * Parameterizable tuning policy type for AgentRadixSortUpsweep
- */
-template <
-    int                 _BLOCK_THREADS,     ///< Threads per thread block
-    int                 _ITEMS_PER_THREAD,  ///< Items per thread (per tile of input)
-    CacheLoadModifier   _LOAD_MODIFIER,     ///< Cache load modifier for reading keys
-    int                 _RADIX_BITS>        ///< The number of radix bits, i.e., log2(bins)
-struct AgentRadixSortUpsweepPolicy
-{
-    enum
-    {
-        BLOCK_THREADS       = _BLOCK_THREADS,       ///< Threads per thread block
-        ITEMS_PER_THREAD    = _ITEMS_PER_THREAD,    ///< Items per thread (per tile of input)
-        RADIX_BITS          = _RADIX_BITS,          ///< The number of radix bits, i.e., log2(bins)
-    };
-
-    static const CacheLoadModifier LOAD_MODIFIER = _LOAD_MODIFIER;      ///< Cache load modifier for reading keys
-};
-
-
-/******************************************************************************
- * Thread block abstractions
- ******************************************************************************/
-
-/**
- * \brief AgentRadixSortUpsweep implements a stateful abstraction of CUDA thread blocks for participating in device-wide radix sort upsweep .
- */
-template <
-    typename AgentRadixSortUpsweepPolicy,   ///< Parameterized AgentRadixSortUpsweepPolicy tuning policy type
-    typename KeyT,                          ///< KeyT type
-    typename OffsetT>                       ///< Signed integer type for global offsets
-struct AgentRadixSortUpsweep
-{
-
-    //---------------------------------------------------------------------
-    // Type definitions and constants
-    //---------------------------------------------------------------------
-
-    typedef typename Traits<KeyT>::UnsignedBits UnsignedBits;
-
-    // Integer type for digit counters (to be packed into words of PackedCounters)
-    typedef unsigned char DigitCounter;
-
-    // Integer type for packing DigitCounters into columns of shared memory banks
-    typedef unsigned int PackedCounter;
-
-    static const CacheLoadModifier LOAD_MODIFIER = AgentRadixSortUpsweepPolicy::LOAD_MODIFIER;
-
-    enum
-    {
-        RADIX_BITS              = AgentRadixSortUpsweepPolicy::RADIX_BITS,
-        BLOCK_THREADS           = AgentRadixSortUpsweepPolicy::BLOCK_THREADS,
-        KEYS_PER_THREAD         = AgentRadixSortUpsweepPolicy::ITEMS_PER_THREAD,
-
-        RADIX_DIGITS            = 1 << RADIX_BITS,
-
-        LOG_WARP_THREADS        = CUB_PTX_LOG_WARP_THREADS,
-        WARP_THREADS            = 1 << LOG_WARP_THREADS,
-        WARPS                   = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS,
-
-        TILE_ITEMS              = BLOCK_THREADS * KEYS_PER_THREAD,
-
-        BYTES_PER_COUNTER       = sizeof(DigitCounter),
-        LOG_BYTES_PER_COUNTER   = Log2<BYTES_PER_COUNTER>::VALUE,
-
-        PACKING_RATIO           = sizeof(PackedCounter) / sizeof(DigitCounter),
-        LOG_PACKING_RATIO       = Log2<PACKING_RATIO>::VALUE,
-
-        LOG_COUNTER_LANES       = CUB_MAX(0, RADIX_BITS - LOG_PACKING_RATIO),
-        COUNTER_LANES           = 1 << LOG_COUNTER_LANES,
-
-        // To prevent counter overflow, we must periodically unpack and aggregate the
-        // digit counters back into registers.  Each counter lane is assigned to a
-        // warp for aggregation.
-
-        LANES_PER_WARP          = CUB_MAX(1, (COUNTER_LANES + WARPS - 1) / WARPS),
-
-        // Unroll tiles in batches without risk of counter overflow
-        UNROLL_COUNT            = CUB_MIN(64, 255 / KEYS_PER_THREAD),
-        UNROLLED_ELEMENTS       = UNROLL_COUNT * TILE_ITEMS,
-    };
-
-
-    // Input iterator wrapper type (for applying cache modifier)s
-    typedef CacheModifiedInputIterator<LOAD_MODIFIER, UnsignedBits, OffsetT> KeysItr;
-
-    /**
-     * Shared memory storage layout
-     */
-    union __align__(16) _TempStorage
-    {
-        DigitCounter    thread_counters[COUNTER_LANES][BLOCK_THREADS][PACKING_RATIO];
-        PackedCounter   packed_thread_counters[COUNTER_LANES][BLOCK_THREADS];
-        OffsetT         block_counters[WARP_THREADS][RADIX_DIGITS];
-    };
-
-
-    /// Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    //---------------------------------------------------------------------
-    // Thread fields (aggregate state bundle)
-    //---------------------------------------------------------------------
-
-    // Shared storage for this CTA
-    _TempStorage    &temp_storage;
-
-    // Thread-local counters for periodically aggregating composite-counter lanes
-    OffsetT         local_counts[LANES_PER_WARP][PACKING_RATIO];
-
-    // Input and output device pointers
-    KeysItr         d_keys_in;
-
-    // The least-significant bit position of the current digit to extract
-    int             current_bit;
-
-    // Number of bits in current digit
-    int             num_bits;
-
-
-
-    //---------------------------------------------------------------------
-    // Helper structure for templated iteration
-    //---------------------------------------------------------------------
-
-    // Iterate
-    template <int COUNT, int MAX>
-    struct Iterate
-    {
-        // BucketKeys
-        static __device__ __forceinline__ void BucketKeys(
-            AgentRadixSortUpsweep       &cta,
-            UnsignedBits                keys[KEYS_PER_THREAD])
-        {
-            cta.Bucket(keys[COUNT]);
-
-            // Next
-            Iterate<COUNT + 1, MAX>::BucketKeys(cta, keys);
-        }
-    };
-
-    // Terminate
-    template <int MAX>
-    struct Iterate<MAX, MAX>
-    {
-        // BucketKeys
-        static __device__ __forceinline__ void BucketKeys(AgentRadixSortUpsweep &/*cta*/, UnsignedBits /*keys*/[KEYS_PER_THREAD]) {}
-    };
-
-
-    //---------------------------------------------------------------------
-    // Utility methods
-    //---------------------------------------------------------------------
-
-    /**
-     * Decode a key and increment corresponding smem digit counter
-     */
-    __device__ __forceinline__ void Bucket(UnsignedBits key)
-    {
-        // Perform transform op
-        UnsignedBits converted_key = Traits<KeyT>::TwiddleIn(key);
-
-        // Extract current digit bits
-        UnsignedBits digit = BFE(converted_key, current_bit, num_bits);
-
-        // Get sub-counter offset
-        UnsignedBits sub_counter = digit & (PACKING_RATIO - 1);
-
-        // Get row offset
-        UnsignedBits row_offset = digit >> LOG_PACKING_RATIO;
-
-        // Increment counter
-        temp_storage.thread_counters[row_offset][threadIdx.x][sub_counter]++;
-    }
-
-
-    /**
-     * Reset composite counters
-     */
-    __device__ __forceinline__ void ResetDigitCounters()
-    {
-        #pragma unroll
-        for (int LANE = 0; LANE < COUNTER_LANES; LANE++)
-        {
-            temp_storage.packed_thread_counters[LANE][threadIdx.x] = 0;
-        }
-    }
-
-
-    /**
-     * Reset the unpacked counters in each thread
-     */
-    __device__ __forceinline__ void ResetUnpackedCounters()
-    {
-        #pragma unroll
-        for (int LANE = 0; LANE < LANES_PER_WARP; LANE++)
-        {
-            #pragma unroll
-            for (int UNPACKED_COUNTER = 0; UNPACKED_COUNTER < PACKING_RATIO; UNPACKED_COUNTER++)
-            {
-                local_counts[LANE][UNPACKED_COUNTER] = 0;
-            }
-        }
-    }
-
-
-    /**
-     * Extracts and aggregates the digit counters for each counter lane
-     * owned by this warp
-     */
-    __device__ __forceinline__ void UnpackDigitCounts()
-    {
-        unsigned int warp_id = threadIdx.x >> LOG_WARP_THREADS;
-        unsigned int warp_tid = LaneId();
-
-        #pragma unroll
-        for (int LANE = 0; LANE < LANES_PER_WARP; LANE++)
-        {
-            const int counter_lane = (LANE * WARPS) + warp_id;
-            if (counter_lane < COUNTER_LANES)
-            {
-                #pragma unroll
-                for (int PACKED_COUNTER = 0; PACKED_COUNTER < BLOCK_THREADS; PACKED_COUNTER += WARP_THREADS)
-                {
-                    #pragma unroll
-                    for (int UNPACKED_COUNTER = 0; UNPACKED_COUNTER < PACKING_RATIO; UNPACKED_COUNTER++)
-                    {
-                        OffsetT counter = temp_storage.thread_counters[counter_lane][warp_tid + PACKED_COUNTER][UNPACKED_COUNTER];
-                        local_counts[LANE][UNPACKED_COUNTER] += counter;
-                    }
-                }
-            }
-        }
-    }
-
-
-    /**
-     * Processes a single, full tile
-     */
-    __device__ __forceinline__ void ProcessFullTile(OffsetT block_offset)
-    {
-        // Tile of keys
-        UnsignedBits keys[KEYS_PER_THREAD];
-
-        LoadDirectStriped<BLOCK_THREADS>(threadIdx.x, d_keys_in + block_offset, keys);
-
-        // Prevent hoisting
-        CTA_SYNC();
-
-        // Bucket tile of keys
-        Iterate<0, KEYS_PER_THREAD>::BucketKeys(*this, keys);
-    }
-
-
-    /**
-     * Processes a single load (may have some threads masked off)
-     */
-    __device__ __forceinline__ void ProcessPartialTile(
-        OffsetT block_offset,
-        const OffsetT &block_end)
-    {
-        // Process partial tile if necessary using single loads
-        block_offset += threadIdx.x;
-        while (block_offset < block_end)
-        {
-            // Load and bucket key
-            UnsignedBits key = d_keys_in[block_offset];
-            Bucket(key);
-            block_offset += BLOCK_THREADS;
-        }
-    }
-
-
-    //---------------------------------------------------------------------
-    // Interface
-    //---------------------------------------------------------------------
-
-    /**
-     * Constructor
-     */
-    __device__ __forceinline__ AgentRadixSortUpsweep(
-        TempStorage &temp_storage,
-        const KeyT  *d_keys_in,
-        int         current_bit,
-        int         num_bits)
-    :
-        temp_storage(temp_storage.Alias()),
-        d_keys_in(reinterpret_cast<const UnsignedBits*>(d_keys_in)),
-        current_bit(current_bit),
-        num_bits(num_bits)
-    {}
-
-
-    /**
-     * Compute radix digit histograms from a segment of input tiles.
-     */
-    __device__ __forceinline__ void ProcessRegion(
-        OffsetT          block_offset,
-        const OffsetT    &block_end)
-    {
-        // Reset digit counters in smem and unpacked counters in registers
-        ResetDigitCounters();
-        ResetUnpackedCounters();
-
-        // Unroll batches of full tiles
-        while (block_offset + UNROLLED_ELEMENTS <= block_end)
-        {
-            for (int i = 0; i < UNROLL_COUNT; ++i)
-            {
-                ProcessFullTile(block_offset);
-                block_offset += TILE_ITEMS;
-            }
-
-            CTA_SYNC();
-
-            // Aggregate back into local_count registers to prevent overflow
-            UnpackDigitCounts();
-
-            CTA_SYNC();
-
-            // Reset composite counters in lanes
-            ResetDigitCounters();
-        }
-
-        // Unroll single full tiles
-        while (block_offset + TILE_ITEMS <= block_end)
-        {
-            ProcessFullTile(block_offset);
-            block_offset += TILE_ITEMS;
-        }
-
-        // Process partial tile if necessary
-        ProcessPartialTile(
-            block_offset,
-            block_end);
-
-        CTA_SYNC();
-
-        // Aggregate back into local_count registers
-        UnpackDigitCounts();
-    }
-
-
-    /**
-     * Extract counts (saving them to the external array)
-     */
-    template <bool IS_DESCENDING>
-    __device__ __forceinline__ void ExtractCounts(
-        OffsetT     *counters,
-        int         bin_stride = 1,
-        int         bin_offset = 0)
-    {
-        unsigned int warp_id    = threadIdx.x >> LOG_WARP_THREADS;
-        unsigned int warp_tid   = LaneId();
-
-        // Place unpacked digit counters in shared memory
-        #pragma unroll
-        for (int LANE = 0; LANE < LANES_PER_WARP; LANE++)
-        {
-            int counter_lane = (LANE * WARPS) + warp_id;
-            if (counter_lane < COUNTER_LANES)
-            {
-                int digit_row = counter_lane << LOG_PACKING_RATIO;
-
-                #pragma unroll
-                for (int UNPACKED_COUNTER = 0; UNPACKED_COUNTER < PACKING_RATIO; UNPACKED_COUNTER++)
-                {
-                    int bin_idx = digit_row + UNPACKED_COUNTER;
-
-                    temp_storage.block_counters[warp_tid][bin_idx] =
-                        local_counts[LANE][UNPACKED_COUNTER];
-                }
-            }
-        }
-
-        CTA_SYNC();
-
-        // Rake-reduce bin_count reductions
-
-        // Whole blocks
-        #pragma unroll
-        for (int BIN_BASE   = RADIX_DIGITS % BLOCK_THREADS;
-            (BIN_BASE + BLOCK_THREADS) <= RADIX_DIGITS;
-            BIN_BASE += BLOCK_THREADS)
-        {
-            int bin_idx = BIN_BASE + threadIdx.x;
-
-            OffsetT bin_count = 0;
-            #pragma unroll
-            for (int i = 0; i < WARP_THREADS; ++i)
-                bin_count += temp_storage.block_counters[i][bin_idx];
-
-            if (IS_DESCENDING)
-                bin_idx = RADIX_DIGITS - bin_idx - 1;
-
-            counters[(bin_stride * bin_idx) + bin_offset] = bin_count;
-        }
-
-        // Remainder
-        if ((RADIX_DIGITS % BLOCK_THREADS != 0) && (threadIdx.x < RADIX_DIGITS))
-        {
-            int bin_idx = threadIdx.x;
-
-            OffsetT bin_count = 0;
-            #pragma unroll
-            for (int i = 0; i < WARP_THREADS; ++i)
-                bin_count += temp_storage.block_counters[i][bin_idx];
-
-            if (IS_DESCENDING)
-                bin_idx = RADIX_DIGITS - bin_idx - 1;
-
-            counters[(bin_stride * bin_idx) + bin_offset] = bin_count;
-        }
-    }
-
-
-    /**
-     * Extract counts
-     */
-    template <int BINS_TRACKED_PER_THREAD>
-    __device__ __forceinline__ void ExtractCounts(
-        OffsetT (&bin_count)[BINS_TRACKED_PER_THREAD])  ///< [out] The exclusive prefix sum for the digits [(threadIdx.x * BINS_TRACKED_PER_THREAD) ... (threadIdx.x * BINS_TRACKED_PER_THREAD) + BINS_TRACKED_PER_THREAD - 1]
-    {
-        unsigned int warp_id    = threadIdx.x >> LOG_WARP_THREADS;
-        unsigned int warp_tid   = LaneId();
-
-        // Place unpacked digit counters in shared memory
-        #pragma unroll
-        for (int LANE = 0; LANE < LANES_PER_WARP; LANE++)
-        {
-            int counter_lane = (LANE * WARPS) + warp_id;
-            if (counter_lane < COUNTER_LANES)
-            {
-                int digit_row = counter_lane << LOG_PACKING_RATIO;
-
-                #pragma unroll
-                for (int UNPACKED_COUNTER = 0; UNPACKED_COUNTER < PACKING_RATIO; UNPACKED_COUNTER++)
-                {
-                    int bin_idx = digit_row + UNPACKED_COUNTER;
-
-                    temp_storage.block_counters[warp_tid][bin_idx] =
-                        local_counts[LANE][UNPACKED_COUNTER];
-                }
-            }
-        }
-
-        CTA_SYNC();
-
-        // Rake-reduce bin_count reductions
-        #pragma unroll
-        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
-        {
-            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
-
-            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
-            {
-                bin_count[track] = 0;
-
-                #pragma unroll
-                for (int i = 0; i < WARP_THREADS; ++i)
-                    bin_count[track] += temp_storage.block_counters[i][bin_idx];
-            }
-        }
-    }
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/agent/agent_reduce.cuh b/pfsimulator/third_party/cub/agent/agent_reduce.cuh
deleted file mode 100644
index 000a905cc..000000000
--- a/pfsimulator/third_party/cub/agent/agent_reduce.cuh
+++ /dev/null
@@ -1,385 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::AgentReduce implements a stateful abstraction of CUDA thread blocks for participating in device-wide reduction .
- */
-
-#pragma once
-
-#include <iterator>
-
-#include "../block/block_load.cuh"
-#include "../block/block_reduce.cuh"
-#include "../grid/grid_mapping.cuh"
-#include "../grid/grid_even_share.cuh"
-#include "../util_type.cuh"
-#include "../iterator/cache_modified_input_iterator.cuh"
-#include "../util_namespace.cuh"
-
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/******************************************************************************
- * Tuning policy types
- ******************************************************************************/
-
-/**
- * Parameterizable tuning policy type for AgentReduce
- */
-template <
-    int                     _BLOCK_THREADS,         ///< Threads per thread block
-    int                     _ITEMS_PER_THREAD,      ///< Items per thread (per tile of input)
-    int                     _VECTOR_LOAD_LENGTH,    ///< Number of items per vectorized load
-    BlockReduceAlgorithm    _BLOCK_ALGORITHM,       ///< Cooperative block-wide reduction algorithm to use
-    CacheLoadModifier       _LOAD_MODIFIER>         ///< Cache load modifier for reading input elements
-struct AgentReducePolicy
-{
-    enum
-    {
-        BLOCK_THREADS       = _BLOCK_THREADS,       ///< Threads per thread block
-        ITEMS_PER_THREAD    = _ITEMS_PER_THREAD,    ///< Items per thread (per tile of input)
-        VECTOR_LOAD_LENGTH  = _VECTOR_LOAD_LENGTH,  ///< Number of items per vectorized load
-    };
-
-    static const BlockReduceAlgorithm  BLOCK_ALGORITHM      = _BLOCK_ALGORITHM;     ///< Cooperative block-wide reduction algorithm to use
-    static const CacheLoadModifier     LOAD_MODIFIER        = _LOAD_MODIFIER;       ///< Cache load modifier for reading input elements
-};
-
-
-
-/******************************************************************************
- * Thread block abstractions
- ******************************************************************************/
-
-/**
- * \brief AgentReduce implements a stateful abstraction of CUDA thread blocks for participating in device-wide reduction .
- *
- * Each thread reduces only the values it loads. If \p FIRST_TILE, this
- * partial reduction is stored into \p thread_aggregate.  Otherwise it is
- * accumulated into \p thread_aggregate.
- */
-template <
-    typename AgentReducePolicy,        ///< Parameterized AgentReducePolicy tuning policy type
-    typename InputIteratorT,           ///< Random-access iterator type for input
-    typename OutputIteratorT,          ///< Random-access iterator type for output
-    typename OffsetT,                  ///< Signed integer type for global offsets
-    typename ReductionOp>              ///< Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
-struct AgentReduce
-{
-
-    //---------------------------------------------------------------------
-    // Types and constants
-    //---------------------------------------------------------------------
-
-    /// The input value type
-    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
-
-    /// The output value type
-    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
-        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
-        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
-
-    /// Vector type of InputT for data movement
-    typedef typename CubVector<InputT, AgentReducePolicy::VECTOR_LOAD_LENGTH>::Type VectorT;
-
-    /// Input iterator wrapper type (for applying cache modifier)
-    typedef typename If<IsPointer<InputIteratorT>::VALUE,
-            CacheModifiedInputIterator<AgentReducePolicy::LOAD_MODIFIER, InputT, OffsetT>,      // Wrap the native input pointer with CacheModifiedInputIterator
-            InputIteratorT>::Type                                                               // Directly use the supplied input iterator type
-        WrappedInputIteratorT;
-
-    /// Constants
-    enum
-    {
-        BLOCK_THREADS       = AgentReducePolicy::BLOCK_THREADS,
-        ITEMS_PER_THREAD    = AgentReducePolicy::ITEMS_PER_THREAD,
-        VECTOR_LOAD_LENGTH  = CUB_MIN(ITEMS_PER_THREAD, AgentReducePolicy::VECTOR_LOAD_LENGTH),
-        TILE_ITEMS          = BLOCK_THREADS * ITEMS_PER_THREAD,
-
-        // Can vectorize according to the policy if the input iterator is a native pointer to a primitive type
-        ATTEMPT_VECTORIZATION   = (VECTOR_LOAD_LENGTH > 1) &&
-                                    (ITEMS_PER_THREAD % VECTOR_LOAD_LENGTH == 0) &&
-                                    (IsPointer<InputIteratorT>::VALUE) && Traits<InputT>::PRIMITIVE,
-
-    };
-
-    static const CacheLoadModifier    LOAD_MODIFIER   = AgentReducePolicy::LOAD_MODIFIER;
-    static const BlockReduceAlgorithm BLOCK_ALGORITHM = AgentReducePolicy::BLOCK_ALGORITHM;
-
-    /// Parameterized BlockReduce primitive
-    typedef BlockReduce<OutputT, BLOCK_THREADS, AgentReducePolicy::BLOCK_ALGORITHM> BlockReduceT;
-
-    /// Shared memory type required by this thread block
-    struct _TempStorage
-    {
-        typename BlockReduceT::TempStorage  reduce;
-    };
-
-    /// Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    //---------------------------------------------------------------------
-    // Per-thread fields
-    //---------------------------------------------------------------------
-
-    _TempStorage&           temp_storage;       ///< Reference to temp_storage
-    InputIteratorT          d_in;               ///< Input data to reduce
-    WrappedInputIteratorT   d_wrapped_in;       ///< Wrapped input data to reduce
-    ReductionOp             reduction_op;       ///< Binary reduction operator
-
-
-    //---------------------------------------------------------------------
-    // Utility
-    //---------------------------------------------------------------------
-
-
-    // Whether or not the input is aligned with the vector type (specialized for types we can vectorize)
-    template <typename Iterator>
-    static __device__ __forceinline__ bool IsAligned(
-        Iterator        d_in,
-        Int2Type<true>  /*can_vectorize*/)
-    {
-        return (size_t(d_in) & (sizeof(VectorT) - 1)) == 0;
-    }
-
-    // Whether or not the input is aligned with the vector type (specialized for types we cannot vectorize)
-    template <typename Iterator>
-    static __device__ __forceinline__ bool IsAligned(
-        Iterator        /*d_in*/,
-        Int2Type<false> /*can_vectorize*/)
-    {
-        return false;
-    }
-
-
-    //---------------------------------------------------------------------
-    // Constructor
-    //---------------------------------------------------------------------
-
-    /**
-     * Constructor
-     */
-    __device__ __forceinline__ AgentReduce(
-        TempStorage&            temp_storage,       ///< Reference to temp_storage
-        InputIteratorT          d_in,               ///< Input data to reduce
-        ReductionOp             reduction_op)       ///< Binary reduction operator
-    :
-        temp_storage(temp_storage.Alias()),
-        d_in(d_in),
-        d_wrapped_in(d_in),
-        reduction_op(reduction_op)
-    {}
-
-
-    //---------------------------------------------------------------------
-    // Tile consumption
-    //---------------------------------------------------------------------
-
-    /**
-     * Consume a full tile of input (non-vectorized)
-     */
-    template <int IS_FIRST_TILE>
-    __device__ __forceinline__ void ConsumeTile(
-        OutputT                 &thread_aggregate,
-        OffsetT                 block_offset,       ///< The offset the tile to consume
-        int                     /*valid_items*/,    ///< The number of valid items in the tile
-        Int2Type<true>          /*is_full_tile*/,   ///< Whether or not this is a full tile
-        Int2Type<false>         /*can_vectorize*/)  ///< Whether or not we can vectorize loads
-    {
-        OutputT items[ITEMS_PER_THREAD];
-
-        // Load items in striped fashion
-        LoadDirectStriped<BLOCK_THREADS>(threadIdx.x, d_wrapped_in + block_offset, items);
-
-        // Reduce items within each thread stripe
-        thread_aggregate = (IS_FIRST_TILE) ?
-            internal::ThreadReduce(items, reduction_op) :
-            internal::ThreadReduce(items, reduction_op, thread_aggregate);
-    }
-
-
-    /**
-     * Consume a full tile of input (vectorized)
-     */
-    template <int IS_FIRST_TILE>
-    __device__ __forceinline__ void ConsumeTile(
-        OutputT                 &thread_aggregate,
-        OffsetT                 block_offset,       ///< The offset the tile to consume
-        int                     /*valid_items*/,    ///< The number of valid items in the tile
-        Int2Type<true>          /*is_full_tile*/,   ///< Whether or not this is a full tile
-        Int2Type<true>          /*can_vectorize*/)  ///< Whether or not we can vectorize loads
-    {
-        // Alias items as an array of VectorT and load it in striped fashion
-        enum { WORDS =  ITEMS_PER_THREAD / VECTOR_LOAD_LENGTH };
-
-        // Fabricate a vectorized input iterator
-        InputT *d_in_unqualified = const_cast<InputT*>(d_in) + block_offset + (threadIdx.x * VECTOR_LOAD_LENGTH);
-        CacheModifiedInputIterator<AgentReducePolicy::LOAD_MODIFIER, VectorT, OffsetT> d_vec_in(
-            reinterpret_cast<VectorT*>(d_in_unqualified));
-
-        // Load items as vector items
-        InputT input_items[ITEMS_PER_THREAD];
-        VectorT *vec_items = reinterpret_cast<VectorT*>(input_items);
-        #pragma unroll
-        for (int i = 0; i < WORDS; ++i)
-            vec_items[i] = d_vec_in[BLOCK_THREADS * i];
-
-        // Convert from input type to output type
-        OutputT items[ITEMS_PER_THREAD];
-        #pragma unroll
-        for (int i = 0; i < ITEMS_PER_THREAD; ++i)
-            items[i] = input_items[i];
-
-        // Reduce items within each thread stripe
-        thread_aggregate = (IS_FIRST_TILE) ?
-            internal::ThreadReduce(items, reduction_op) :
-            internal::ThreadReduce(items, reduction_op, thread_aggregate);
-    }
-
-
-    /**
-     * Consume a partial tile of input
-     */
-    template <int IS_FIRST_TILE, int CAN_VECTORIZE>
-    __device__ __forceinline__ void ConsumeTile(
-        OutputT                 &thread_aggregate,
-        OffsetT                 block_offset,       ///< The offset the tile to consume
-        int                     valid_items,        ///< The number of valid items in the tile
-        Int2Type<false>         /*is_full_tile*/,   ///< Whether or not this is a full tile
-        Int2Type<CAN_VECTORIZE> /*can_vectorize*/)  ///< Whether or not we can vectorize loads
-    {
-        // Partial tile
-        int thread_offset = threadIdx.x;
-
-        // Read first item
-        if ((IS_FIRST_TILE) && (thread_offset < valid_items))
-        {
-            thread_aggregate = d_wrapped_in[block_offset + thread_offset];
-            thread_offset += BLOCK_THREADS;
-        }
-
-        // Continue reading items (block-striped)
-        while (thread_offset < valid_items)
-        {
-            OutputT item        = d_wrapped_in[block_offset + thread_offset];
-            thread_aggregate    = reduction_op(thread_aggregate, item);
-            thread_offset       += BLOCK_THREADS;
-        }
-    }
-
-
-    //---------------------------------------------------------------
-    // Consume a contiguous segment of tiles
-    //---------------------------------------------------------------------
-
-    /**
-     * \brief Reduce a contiguous segment of input tiles
-     */
-    template <int CAN_VECTORIZE>
-    __device__ __forceinline__ OutputT ConsumeRange(
-        GridEvenShare<OffsetT> &even_share,          ///< GridEvenShare descriptor
-        Int2Type<CAN_VECTORIZE> can_vectorize)      ///< Whether or not we can vectorize loads
-    {
-        OutputT thread_aggregate;
-
-        if (even_share.block_offset + TILE_ITEMS > even_share.block_end)
-        {
-            // First tile isn't full (not all threads have valid items)
-            int valid_items = even_share.block_end - even_share.block_offset;
-            ConsumeTile<true>(thread_aggregate, even_share.block_offset, valid_items, Int2Type<false>(), can_vectorize);
-            return BlockReduceT(temp_storage.reduce).Reduce(thread_aggregate, reduction_op, valid_items);
-        }
-
-        // At least one full block
-        ConsumeTile<true>(thread_aggregate, even_share.block_offset, TILE_ITEMS, Int2Type<true>(), can_vectorize);
-        even_share.block_offset += even_share.block_stride;
-
-        // Consume subsequent full tiles of input
-        while (even_share.block_offset + TILE_ITEMS <= even_share.block_end)
-        {
-            ConsumeTile<false>(thread_aggregate, even_share.block_offset, TILE_ITEMS, Int2Type<true>(), can_vectorize);
-            even_share.block_offset += even_share.block_stride;
-        }
-
-        // Consume a partially-full tile
-        if (even_share.block_offset < even_share.block_end)
-        {
-            int valid_items = even_share.block_end - even_share.block_offset;
-            ConsumeTile<false>(thread_aggregate, even_share.block_offset, valid_items, Int2Type<false>(), can_vectorize);
-        }
-
-        // Compute block-wide reduction (all threads have valid items)
-        return BlockReduceT(temp_storage.reduce).Reduce(thread_aggregate, reduction_op);
-    }
-
-
-    /**
-     * \brief Reduce a contiguous segment of input tiles
-     */
-    __device__ __forceinline__ OutputT ConsumeRange(
-        OffsetT block_offset,                       ///< [in] Threadblock begin offset (inclusive)
-        OffsetT block_end)                          ///< [in] Threadblock end offset (exclusive)
-    {
-        GridEvenShare<OffsetT> even_share;
-        even_share.template BlockInit<TILE_ITEMS>(block_offset, block_end);
-
-        return (IsAligned(d_in + block_offset, Int2Type<ATTEMPT_VECTORIZATION>())) ?
-            ConsumeRange(even_share, Int2Type<true && ATTEMPT_VECTORIZATION>()) :
-            ConsumeRange(even_share, Int2Type<false && ATTEMPT_VECTORIZATION>());
-    }
-
-
-    /**
-     * Reduce a contiguous segment of input tiles
-     */
-    __device__ __forceinline__ OutputT ConsumeTiles(
-        GridEvenShare<OffsetT> &even_share)        ///< [in] GridEvenShare descriptor
-    {
-        // Initialize GRID_MAPPING_STRIP_MINE even-share descriptor for this thread block
-        even_share.template BlockInit<TILE_ITEMS, GRID_MAPPING_STRIP_MINE>();
-
-        return (IsAligned(d_in, Int2Type<ATTEMPT_VECTORIZATION>())) ?
-            ConsumeRange(even_share, Int2Type<true && ATTEMPT_VECTORIZATION>()) :
-            ConsumeRange(even_share, Int2Type<false && ATTEMPT_VECTORIZATION>());
-
-    }
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/agent/agent_reduce_by_key.cuh b/pfsimulator/third_party/cub/agent/agent_reduce_by_key.cuh
deleted file mode 100644
index 51964d3e6..000000000
--- a/pfsimulator/third_party/cub/agent/agent_reduce_by_key.cuh
+++ /dev/null
@@ -1,547 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::AgentReduceByKey implements a stateful abstraction of CUDA thread blocks for participating in device-wide reduce-value-by-key.
- */
-
-#pragma once
-
-#include <iterator>
-
-#include "single_pass_scan_operators.cuh"
-#include "../block/block_load.cuh"
-#include "../block/block_store.cuh"
-#include "../block/block_scan.cuh"
-#include "../block/block_discontinuity.cuh"
-#include "../iterator/cache_modified_input_iterator.cuh"
-#include "../iterator/constant_input_iterator.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/******************************************************************************
- * Tuning policy types
- ******************************************************************************/
-
-/**
- * Parameterizable tuning policy type for AgentReduceByKey
- */
-template <
-    int                         _BLOCK_THREADS,                 ///< Threads per thread block
-    int                         _ITEMS_PER_THREAD,              ///< Items per thread (per tile of input)
-    BlockLoadAlgorithm          _LOAD_ALGORITHM,                ///< The BlockLoad algorithm to use
-    CacheLoadModifier           _LOAD_MODIFIER,                 ///< Cache load modifier for reading input elements
-    BlockScanAlgorithm          _SCAN_ALGORITHM>                ///< The BlockScan algorithm to use
-struct AgentReduceByKeyPolicy
-{
-    enum
-    {
-        BLOCK_THREADS           = _BLOCK_THREADS,               ///< Threads per thread block
-        ITEMS_PER_THREAD        = _ITEMS_PER_THREAD,            ///< Items per thread (per tile of input)
-    };
-
-    static const BlockLoadAlgorithm     LOAD_ALGORITHM          = _LOAD_ALGORITHM;      ///< The BlockLoad algorithm to use
-    static const CacheLoadModifier      LOAD_MODIFIER           = _LOAD_MODIFIER;       ///< Cache load modifier for reading input elements
-    static const BlockScanAlgorithm     SCAN_ALGORITHM          = _SCAN_ALGORITHM;      ///< The BlockScan algorithm to use
-};
-
-
-/******************************************************************************
- * Thread block abstractions
- ******************************************************************************/
-
-/**
- * \brief AgentReduceByKey implements a stateful abstraction of CUDA thread blocks for participating in device-wide reduce-value-by-key
- */
-template <
-    typename    AgentReduceByKeyPolicyT,        ///< Parameterized AgentReduceByKeyPolicy tuning policy type
-    typename    KeysInputIteratorT,             ///< Random-access input iterator type for keys
-    typename    UniqueOutputIteratorT,          ///< Random-access output iterator type for keys
-    typename    ValuesInputIteratorT,           ///< Random-access input iterator type for values
-    typename    AggregatesOutputIteratorT,      ///< Random-access output iterator type for values
-    typename    NumRunsOutputIteratorT,         ///< Output iterator type for recording number of items selected
-    typename    EqualityOpT,                    ///< KeyT equality operator type
-    typename    ReductionOpT,                   ///< ValueT reduction operator type
-    typename    OffsetT>                        ///< Signed integer type for global offsets
-struct AgentReduceByKey
-{
-    //---------------------------------------------------------------------
-    // Types and constants
-    //---------------------------------------------------------------------
-
-    // The input keys type
-    typedef typename std::iterator_traits<KeysInputIteratorT>::value_type KeyInputT;
-
-    // The output keys type
-    typedef typename If<(Equals<typename std::iterator_traits<UniqueOutputIteratorT>::value_type, void>::VALUE),    // KeyOutputT =  (if output iterator's value type is void) ?
-        typename std::iterator_traits<KeysInputIteratorT>::value_type,                                              // ... then the input iterator's value type,
-        typename std::iterator_traits<UniqueOutputIteratorT>::value_type>::Type KeyOutputT;                         // ... else the output iterator's value type
-
-    // The input values type
-    typedef typename std::iterator_traits<ValuesInputIteratorT>::value_type ValueInputT;
-
-    // The output values type
-    typedef typename If<(Equals<typename std::iterator_traits<AggregatesOutputIteratorT>::value_type, void>::VALUE),    // ValueOutputT =  (if output iterator's value type is void) ?
-        typename std::iterator_traits<ValuesInputIteratorT>::value_type,                                                // ... then the input iterator's value type,
-        typename std::iterator_traits<AggregatesOutputIteratorT>::value_type>::Type ValueOutputT;                       // ... else the output iterator's value type
-
-    // Tuple type for scanning (pairs accumulated segment-value with segment-index)
-    typedef KeyValuePair<OffsetT, ValueOutputT> OffsetValuePairT;
-
-    // Tuple type for pairing keys and values
-    typedef KeyValuePair<KeyOutputT, ValueOutputT> KeyValuePairT;
-
-    // Tile status descriptor interface type
-    typedef ReduceByKeyScanTileState<ValueOutputT, OffsetT> ScanTileStateT;
-
-    // Guarded inequality functor
-    template <typename _EqualityOpT>
-    struct GuardedInequalityWrapper
-    {
-        _EqualityOpT     op;             ///< Wrapped equality operator
-        int             num_remaining;  ///< Items remaining
-
-        /// Constructor
-        __host__ __device__ __forceinline__
-        GuardedInequalityWrapper(_EqualityOpT op, int num_remaining) : op(op), num_remaining(num_remaining) {}
-
-        /// Boolean inequality operator, returns <tt>(a != b)</tt>
-        template <typename T>
-        __host__ __device__ __forceinline__ bool operator()(const T &a, const T &b, int idx) const
-        {
-            if (idx < num_remaining)
-                return !op(a, b);   // In bounds
-
-            // Return true if first out-of-bounds item, false otherwise
-            return (idx == num_remaining);
-       }
-    };
-
-
-    // Constants
-    enum
-    {
-        BLOCK_THREADS       = AgentReduceByKeyPolicyT::BLOCK_THREADS,
-        ITEMS_PER_THREAD    = AgentReduceByKeyPolicyT::ITEMS_PER_THREAD,
-        TILE_ITEMS          = BLOCK_THREADS * ITEMS_PER_THREAD,
-        TWO_PHASE_SCATTER   = (ITEMS_PER_THREAD > 1),
-
-        // Whether or not the scan operation has a zero-valued identity value (true if we're performing addition on a primitive type)
-        HAS_IDENTITY_ZERO   = (Equals<ReductionOpT, cub::Sum>::VALUE) && (Traits<ValueOutputT>::PRIMITIVE),
-    };
-
-    // Cache-modified Input iterator wrapper type (for applying cache modifier) for keys
-    typedef typename If<IsPointer<KeysInputIteratorT>::VALUE,
-            CacheModifiedInputIterator<AgentReduceByKeyPolicyT::LOAD_MODIFIER, KeyInputT, OffsetT>,     // Wrap the native input pointer with CacheModifiedValuesInputIterator
-            KeysInputIteratorT>::Type                                                                   // Directly use the supplied input iterator type
-        WrappedKeysInputIteratorT;
-
-    // Cache-modified Input iterator wrapper type (for applying cache modifier) for values
-    typedef typename If<IsPointer<ValuesInputIteratorT>::VALUE,
-            CacheModifiedInputIterator<AgentReduceByKeyPolicyT::LOAD_MODIFIER, ValueInputT, OffsetT>,   // Wrap the native input pointer with CacheModifiedValuesInputIterator
-            ValuesInputIteratorT>::Type                                                                 // Directly use the supplied input iterator type
-        WrappedValuesInputIteratorT;
-
-    // Cache-modified Input iterator wrapper type (for applying cache modifier) for fixup values
-    typedef typename If<IsPointer<AggregatesOutputIteratorT>::VALUE,
-            CacheModifiedInputIterator<AgentReduceByKeyPolicyT::LOAD_MODIFIER, ValueInputT, OffsetT>,   // Wrap the native input pointer with CacheModifiedValuesInputIterator
-            AggregatesOutputIteratorT>::Type                                                            // Directly use the supplied input iterator type
-        WrappedFixupInputIteratorT;
-
-    // Reduce-value-by-segment scan operator
-    typedef ReduceBySegmentOp<ReductionOpT> ReduceBySegmentOpT;
-
-    // Parameterized BlockLoad type for keys
-    typedef BlockLoad<
-            KeyOutputT,
-            BLOCK_THREADS,
-            ITEMS_PER_THREAD,
-            AgentReduceByKeyPolicyT::LOAD_ALGORITHM>
-        BlockLoadKeysT;
-
-    // Parameterized BlockLoad type for values
-    typedef BlockLoad<
-            ValueOutputT,
-            BLOCK_THREADS,
-            ITEMS_PER_THREAD,
-            AgentReduceByKeyPolicyT::LOAD_ALGORITHM>
-        BlockLoadValuesT;
-
-    // Parameterized BlockDiscontinuity type for keys
-    typedef BlockDiscontinuity<
-            KeyOutputT,
-            BLOCK_THREADS>
-        BlockDiscontinuityKeys;
-
-    // Parameterized BlockScan type
-    typedef BlockScan<
-            OffsetValuePairT,
-            BLOCK_THREADS,
-            AgentReduceByKeyPolicyT::SCAN_ALGORITHM>
-        BlockScanT;
-
-    // Callback type for obtaining tile prefix during block scan
-    typedef TilePrefixCallbackOp<
-            OffsetValuePairT,
-            ReduceBySegmentOpT,
-            ScanTileStateT>
-        TilePrefixCallbackOpT;
-
-    // Key and value exchange types
-    typedef KeyOutputT    KeyExchangeT[TILE_ITEMS + 1];
-    typedef ValueOutputT  ValueExchangeT[TILE_ITEMS + 1];
-
-    // Shared memory type for this thread block
-    union _TempStorage
-    {
-        struct
-        {
-            typename BlockScanT::TempStorage                scan;           // Smem needed for tile scanning
-            typename TilePrefixCallbackOpT::TempStorage     prefix;         // Smem needed for cooperative prefix callback
-            typename BlockDiscontinuityKeys::TempStorage    discontinuity;  // Smem needed for discontinuity detection
-        };
-
-        // Smem needed for loading keys
-        typename BlockLoadKeysT::TempStorage load_keys;
-
-        // Smem needed for loading values
-        typename BlockLoadValuesT::TempStorage load_values;
-
-        // Smem needed for compacting key value pairs(allows non POD items in this union)
-        Uninitialized<KeyValuePairT[TILE_ITEMS + 1]> raw_exchange;
-    };
-
-    // Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    //---------------------------------------------------------------------
-    // Per-thread fields
-    //---------------------------------------------------------------------
-
-    _TempStorage&                   temp_storage;       ///< Reference to temp_storage
-    WrappedKeysInputIteratorT       d_keys_in;          ///< Input keys
-    UniqueOutputIteratorT           d_unique_out;       ///< Unique output keys
-    WrappedValuesInputIteratorT     d_values_in;        ///< Input values
-    AggregatesOutputIteratorT       d_aggregates_out;   ///< Output value aggregates
-    NumRunsOutputIteratorT          d_num_runs_out;     ///< Output pointer for total number of segments identified
-    EqualityOpT                     equality_op;        ///< KeyT equality operator
-    ReductionOpT                    reduction_op;       ///< Reduction operator
-    ReduceBySegmentOpT              scan_op;            ///< Reduce-by-segment scan operator
-
-
-    //---------------------------------------------------------------------
-    // Constructor
-    //---------------------------------------------------------------------
-
-    // Constructor
-    __device__ __forceinline__
-    AgentReduceByKey(
-        TempStorage&                temp_storage,       ///< Reference to temp_storage
-        KeysInputIteratorT          d_keys_in,          ///< Input keys
-        UniqueOutputIteratorT       d_unique_out,       ///< Unique output keys
-        ValuesInputIteratorT        d_values_in,        ///< Input values
-        AggregatesOutputIteratorT   d_aggregates_out,   ///< Output value aggregates
-        NumRunsOutputIteratorT      d_num_runs_out,     ///< Output pointer for total number of segments identified
-        EqualityOpT                 equality_op,        ///< KeyT equality operator
-        ReductionOpT                reduction_op)       ///< ValueT reduction operator
-    :
-        temp_storage(temp_storage.Alias()),
-        d_keys_in(d_keys_in),
-        d_unique_out(d_unique_out),
-        d_values_in(d_values_in),
-        d_aggregates_out(d_aggregates_out),
-        d_num_runs_out(d_num_runs_out),
-        equality_op(equality_op),
-        reduction_op(reduction_op),
-        scan_op(reduction_op)
-    {}
-
-
-    //---------------------------------------------------------------------
-    // Scatter utility methods
-    //---------------------------------------------------------------------
-
-    /**
-     * Directly scatter flagged items to output offsets
-     */
-    __device__ __forceinline__ void ScatterDirect(
-        KeyValuePairT   (&scatter_items)[ITEMS_PER_THREAD],
-        OffsetT         (&segment_flags)[ITEMS_PER_THREAD],
-        OffsetT         (&segment_indices)[ITEMS_PER_THREAD])
-    {
-        // Scatter flagged keys and values
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            if (segment_flags[ITEM])
-            {
-                d_unique_out[segment_indices[ITEM]]     = scatter_items[ITEM].key;
-                d_aggregates_out[segment_indices[ITEM]] = scatter_items[ITEM].value;
-            }
-        }
-    }
-
-
-    /**
-     * 2-phase scatter flagged items to output offsets
-     *
-     * The exclusive scan causes each head flag to be paired with the previous
-     * value aggregate: the scatter offsets must be decremented for value aggregates
-     */
-    __device__ __forceinline__ void ScatterTwoPhase(
-        KeyValuePairT   (&scatter_items)[ITEMS_PER_THREAD],
-        OffsetT         (&segment_flags)[ITEMS_PER_THREAD],
-        OffsetT         (&segment_indices)[ITEMS_PER_THREAD],
-        OffsetT         num_tile_segments,
-        OffsetT         num_tile_segments_prefix)
-    {
-        CTA_SYNC();
-
-        // Compact and scatter pairs
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            if (segment_flags[ITEM])
-            {
-                temp_storage.raw_exchange.Alias()[segment_indices[ITEM] - num_tile_segments_prefix] = scatter_items[ITEM];
-            }
-        }
-
-        CTA_SYNC();
-
-        for (int item = threadIdx.x; item < num_tile_segments; item += BLOCK_THREADS)
-        {
-            KeyValuePairT pair                                  = temp_storage.raw_exchange.Alias()[item];
-            d_unique_out[num_tile_segments_prefix + item]       = pair.key;
-            d_aggregates_out[num_tile_segments_prefix + item]   = pair.value;
-        }
-    }
-
-
-    /**
-     * Scatter flagged items
-     */
-    __device__ __forceinline__ void Scatter(
-        KeyValuePairT   (&scatter_items)[ITEMS_PER_THREAD],
-        OffsetT         (&segment_flags)[ITEMS_PER_THREAD],
-        OffsetT         (&segment_indices)[ITEMS_PER_THREAD],
-        OffsetT         num_tile_segments,
-        OffsetT         num_tile_segments_prefix)
-    {
-        // Do a one-phase scatter if (a) two-phase is disabled or (b) the average number of selected items per thread is less than one
-        if (TWO_PHASE_SCATTER && (num_tile_segments > BLOCK_THREADS))
-        {
-            ScatterTwoPhase(
-                scatter_items,
-                segment_flags,
-                segment_indices,
-                num_tile_segments,
-                num_tile_segments_prefix);
-        }
-        else
-        {
-            ScatterDirect(
-                scatter_items,
-                segment_flags,
-                segment_indices);
-        }
-    }
-
-
-    //---------------------------------------------------------------------
-    // Cooperatively scan a device-wide sequence of tiles with other CTAs
-    //---------------------------------------------------------------------
-
-    /**
-     * Process a tile of input (dynamic chained scan)
-     */
-    template <bool IS_LAST_TILE>                ///< Whether the current tile is the last tile
-    __device__ __forceinline__ void ConsumeTile(
-        OffsetT             num_remaining,      ///< Number of global input items remaining (including this tile)
-        int                 tile_idx,           ///< Tile index
-        OffsetT             tile_offset,        ///< Tile offset
-        ScanTileStateT&     tile_state)         ///< Global tile state descriptor
-    {
-        KeyOutputT          keys[ITEMS_PER_THREAD];             // Tile keys
-        KeyOutputT          prev_keys[ITEMS_PER_THREAD];        // Tile keys shuffled up
-        ValueOutputT        values[ITEMS_PER_THREAD];           // Tile values
-        OffsetT             head_flags[ITEMS_PER_THREAD];       // Segment head flags
-        OffsetT             segment_indices[ITEMS_PER_THREAD];  // Segment indices
-        OffsetValuePairT    scan_items[ITEMS_PER_THREAD];       // Zipped values and segment flags|indices
-        KeyValuePairT       scatter_items[ITEMS_PER_THREAD];    // Zipped key value pairs for scattering
-
-        // Load keys
-        if (IS_LAST_TILE)
-            BlockLoadKeysT(temp_storage.load_keys).Load(d_keys_in + tile_offset, keys, num_remaining);
-        else
-            BlockLoadKeysT(temp_storage.load_keys).Load(d_keys_in + tile_offset, keys);
-
-        // Load tile predecessor key in first thread
-        KeyOutputT tile_predecessor;
-        if (threadIdx.x == 0)
-        {
-            tile_predecessor = (tile_idx == 0) ?
-                keys[0] :                       // First tile gets repeat of first item (thus first item will not be flagged as a head)
-                d_keys_in[tile_offset - 1];     // Subsequent tiles get last key from previous tile
-        }
-
-        CTA_SYNC();
-
-        // Load values
-        if (IS_LAST_TILE)
-            BlockLoadValuesT(temp_storage.load_values).Load(d_values_in + tile_offset, values, num_remaining);
-        else
-            BlockLoadValuesT(temp_storage.load_values).Load(d_values_in + tile_offset, values);
-
-        CTA_SYNC();
-
-        // Initialize head-flags and shuffle up the previous keys
-        if (IS_LAST_TILE)
-        {
-            // Use custom flag operator to additionally flag the first out-of-bounds item
-            GuardedInequalityWrapper<EqualityOpT> flag_op(equality_op, num_remaining);
-            BlockDiscontinuityKeys(temp_storage.discontinuity).FlagHeads(
-                head_flags, keys, prev_keys, flag_op, tile_predecessor);
-        }
-        else
-        {
-            InequalityWrapper<EqualityOpT> flag_op(equality_op);
-            BlockDiscontinuityKeys(temp_storage.discontinuity).FlagHeads(
-                head_flags, keys, prev_keys, flag_op, tile_predecessor);
-        }
-
-        // Zip values and head flags
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            scan_items[ITEM].value  = values[ITEM];
-            scan_items[ITEM].key    = head_flags[ITEM];
-        }
-
-        // Perform exclusive tile scan
-        OffsetValuePairT    block_aggregate;        // Inclusive block-wide scan aggregate
-        OffsetT             num_segments_prefix;    // Number of segments prior to this tile
-        OffsetValuePairT    total_aggregate;        // The tile prefix folded with block_aggregate
-        if (tile_idx == 0)
-        {
-            // Scan first tile
-            BlockScanT(temp_storage.scan).ExclusiveScan(scan_items, scan_items, scan_op, block_aggregate);
-            num_segments_prefix     = 0;
-            total_aggregate         = block_aggregate;
-
-            // Update tile status if there are successor tiles
-            if ((!IS_LAST_TILE) && (threadIdx.x == 0))
-                tile_state.SetInclusive(0, block_aggregate);
-        }
-        else
-        {
-            // Scan non-first tile
-            TilePrefixCallbackOpT prefix_op(tile_state, temp_storage.prefix, scan_op, tile_idx);
-            BlockScanT(temp_storage.scan).ExclusiveScan(scan_items, scan_items, scan_op, prefix_op);
-
-            block_aggregate         = prefix_op.GetBlockAggregate();
-            num_segments_prefix     = prefix_op.GetExclusivePrefix().key;
-            total_aggregate         = prefix_op.GetInclusivePrefix();
-        }
-
-        // Rezip scatter items and segment indices
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            scatter_items[ITEM].key     = prev_keys[ITEM];
-            scatter_items[ITEM].value   = scan_items[ITEM].value;
-            segment_indices[ITEM]       = scan_items[ITEM].key;
-        }
-
-        // At this point, each flagged segment head has:
-        //  - The key for the previous segment
-        //  - The reduced value from the previous segment
-        //  - The segment index for the reduced value
-
-        // Scatter flagged keys and values
-        OffsetT num_tile_segments = block_aggregate.key;
-        Scatter(scatter_items, head_flags, segment_indices, num_tile_segments, num_segments_prefix);
-
-        // Last thread in last tile will output final count (and last pair, if necessary)
-        if ((IS_LAST_TILE) && (threadIdx.x == BLOCK_THREADS - 1))
-        {
-            OffsetT num_segments = num_segments_prefix + num_tile_segments;
-
-            // If the last tile is a whole tile, output the final_value
-            if (num_remaining == TILE_ITEMS)
-            {
-                d_unique_out[num_segments]      = keys[ITEMS_PER_THREAD - 1];
-                d_aggregates_out[num_segments]  = total_aggregate.value;
-                num_segments++;
-            }
-
-            // Output the total number of items selected
-            *d_num_runs_out = num_segments;
-        }
-    }
-
-
-    /**
-     * Scan tiles of items as part of a dynamic chained scan
-     */
-    __device__ __forceinline__ void ConsumeRange(
-        int                 num_items,          ///< Total number of input items
-        ScanTileStateT&     tile_state,         ///< Global tile state descriptor
-        int                 start_tile)         ///< The starting tile for the current grid
-    {
-        // Blocks are launched in increasing order, so just assign one tile per block
-        int     tile_idx        = start_tile + blockIdx.x;          // Current tile index
-        OffsetT tile_offset     = OffsetT(TILE_ITEMS) * tile_idx;   // Global offset for the current tile
-        OffsetT num_remaining   = num_items - tile_offset;          // Remaining items (including this tile)
-
-        if (num_remaining > TILE_ITEMS)
-        {
-            // Not last tile
-            ConsumeTile<false>(num_remaining, tile_idx, tile_offset, tile_state);
-        }
-        else if (num_remaining > 0)
-        {
-            // Last tile
-            ConsumeTile<true>(num_remaining, tile_idx, tile_offset, tile_state);
-        }
-    }
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/agent/agent_rle.cuh b/pfsimulator/third_party/cub/agent/agent_rle.cuh
deleted file mode 100644
index cb7a4a652..000000000
--- a/pfsimulator/third_party/cub/agent/agent_rle.cuh
+++ /dev/null
@@ -1,837 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::AgentRle implements a stateful abstraction of CUDA thread blocks for participating in device-wide run-length-encode.
- */
-
-#pragma once
-
-#include <iterator>
-
-#include "single_pass_scan_operators.cuh"
-#include "../block/block_load.cuh"
-#include "../block/block_store.cuh"
-#include "../block/block_scan.cuh"
-#include "../block/block_exchange.cuh"
-#include "../block/block_discontinuity.cuh"
-#include "../grid/grid_queue.cuh"
-#include "../iterator/cache_modified_input_iterator.cuh"
-#include "../iterator/constant_input_iterator.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/******************************************************************************
- * Tuning policy types
- ******************************************************************************/
-
-/**
- * Parameterizable tuning policy type for AgentRle
- */
-template <
-    int                         _BLOCK_THREADS,                 ///< Threads per thread block
-    int                         _ITEMS_PER_THREAD,              ///< Items per thread (per tile of input)
-    BlockLoadAlgorithm          _LOAD_ALGORITHM,                ///< The BlockLoad algorithm to use
-    CacheLoadModifier           _LOAD_MODIFIER,                 ///< Cache load modifier for reading input elements
-    bool                        _STORE_WARP_TIME_SLICING,       ///< Whether or not only one warp's worth of shared memory should be allocated and time-sliced among block-warps during any store-related data transpositions (versus each warp having its own storage)
-    BlockScanAlgorithm          _SCAN_ALGORITHM>                ///< The BlockScan algorithm to use
-struct AgentRlePolicy
-{
-    enum
-    {
-        BLOCK_THREADS           = _BLOCK_THREADS,               ///< Threads per thread block
-        ITEMS_PER_THREAD        = _ITEMS_PER_THREAD,            ///< Items per thread (per tile of input)
-        STORE_WARP_TIME_SLICING = _STORE_WARP_TIME_SLICING,     ///< Whether or not only one warp's worth of shared memory should be allocated and time-sliced among block-warps during any store-related data transpositions (versus each warp having its own storage)
-    };
-
-    static const BlockLoadAlgorithm     LOAD_ALGORITHM          = _LOAD_ALGORITHM;      ///< The BlockLoad algorithm to use
-    static const CacheLoadModifier      LOAD_MODIFIER           = _LOAD_MODIFIER;       ///< Cache load modifier for reading input elements
-    static const BlockScanAlgorithm     SCAN_ALGORITHM          = _SCAN_ALGORITHM;      ///< The BlockScan algorithm to use
-};
-
-
-
-
-
-/******************************************************************************
- * Thread block abstractions
- ******************************************************************************/
-
-/**
- * \brief AgentRle implements a stateful abstraction of CUDA thread blocks for participating in device-wide run-length-encode 
- */
-template <
-    typename    AgentRlePolicyT,        ///< Parameterized AgentRlePolicyT tuning policy type
-    typename    InputIteratorT,         ///< Random-access input iterator type for data
-    typename    OffsetsOutputIteratorT, ///< Random-access output iterator type for offset values
-    typename    LengthsOutputIteratorT, ///< Random-access output iterator type for length values
-    typename    EqualityOpT,            ///< T equality operator type
-    typename    OffsetT>                ///< Signed integer type for global offsets
-struct AgentRle
-{
-    //---------------------------------------------------------------------
-    // Types and constants
-    //---------------------------------------------------------------------
-
-    /// The input value type
-    typedef typename std::iterator_traits<InputIteratorT>::value_type T;
-
-    /// The lengths output value type
-    typedef typename If<(Equals<typename std::iterator_traits<LengthsOutputIteratorT>::value_type, void>::VALUE),   // LengthT =  (if output iterator's value type is void) ?
-        OffsetT,                                                                                                    // ... then the OffsetT type,
-        typename std::iterator_traits<LengthsOutputIteratorT>::value_type>::Type LengthT;                           // ... else the output iterator's value type
-
-    /// Tuple type for scanning (pairs run-length and run-index)
-    typedef KeyValuePair<OffsetT, LengthT> LengthOffsetPair;
-
-    /// Tile status descriptor interface type
-    typedef ReduceByKeyScanTileState<LengthT, OffsetT> ScanTileStateT;
-
-    // Constants
-    enum
-    {
-        WARP_THREADS            = CUB_WARP_THREADS(PTX_ARCH),
-        BLOCK_THREADS           = AgentRlePolicyT::BLOCK_THREADS,
-        ITEMS_PER_THREAD        = AgentRlePolicyT::ITEMS_PER_THREAD,
-        WARP_ITEMS              = WARP_THREADS * ITEMS_PER_THREAD,
-        TILE_ITEMS              = BLOCK_THREADS * ITEMS_PER_THREAD,
-        WARPS                   = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS,
-
-        /// Whether or not to sync after loading data
-        SYNC_AFTER_LOAD         = (AgentRlePolicyT::LOAD_ALGORITHM != BLOCK_LOAD_DIRECT),
-
-        /// Whether or not only one warp's worth of shared memory should be allocated and time-sliced among block-warps during any store-related data transpositions (versus each warp having its own storage)
-        STORE_WARP_TIME_SLICING = AgentRlePolicyT::STORE_WARP_TIME_SLICING,
-        ACTIVE_EXCHANGE_WARPS   = (STORE_WARP_TIME_SLICING) ? 1 : WARPS,
-    };
-
-
-    /**
-     * Special operator that signals all out-of-bounds items are not equal to everything else,
-     * forcing both (1) the last item to be tail-flagged and (2) all oob items to be marked
-     * trivial.
-     */
-    template <bool LAST_TILE>
-    struct OobInequalityOp
-    {
-        OffsetT         num_remaining;
-        EqualityOpT      equality_op;
-
-        __device__ __forceinline__ OobInequalityOp(
-            OffsetT     num_remaining,
-            EqualityOpT  equality_op)
-        :
-            num_remaining(num_remaining),
-            equality_op(equality_op)
-        {}
-
-        template <typename Index>
-        __host__ __device__ __forceinline__ bool operator()(T first, T second, Index idx)
-        {
-            if (!LAST_TILE || (idx < num_remaining))
-                return !equality_op(first, second);
-            else
-                return true;
-        }
-    };
-
-
-    // Cache-modified Input iterator wrapper type (for applying cache modifier) for data
-    typedef typename If<IsPointer<InputIteratorT>::VALUE,
-            CacheModifiedInputIterator<AgentRlePolicyT::LOAD_MODIFIER, T, OffsetT>,      // Wrap the native input pointer with CacheModifiedVLengthnputIterator
-            InputIteratorT>::Type                                                       // Directly use the supplied input iterator type
-        WrappedInputIteratorT;
-
-    // Parameterized BlockLoad type for data
-    typedef BlockLoad<
-            T,
-            AgentRlePolicyT::BLOCK_THREADS,
-            AgentRlePolicyT::ITEMS_PER_THREAD,
-            AgentRlePolicyT::LOAD_ALGORITHM>
-        BlockLoadT;
-
-    // Parameterized BlockDiscontinuity type for data
-    typedef BlockDiscontinuity<T, BLOCK_THREADS> BlockDiscontinuityT;
-
-    // Parameterized WarpScan type
-    typedef WarpScan<LengthOffsetPair> WarpScanPairs;
-
-    // Reduce-length-by-run scan operator
-    typedef ReduceBySegmentOp<cub::Sum> ReduceBySegmentOpT;
-
-    // Callback type for obtaining tile prefix during block scan
-    typedef TilePrefixCallbackOp<
-            LengthOffsetPair,
-            ReduceBySegmentOpT,
-            ScanTileStateT>
-        TilePrefixCallbackOpT;
-
-    // Warp exchange types
-    typedef WarpExchange<LengthOffsetPair, ITEMS_PER_THREAD>        WarpExchangePairs;
-
-    typedef typename If<STORE_WARP_TIME_SLICING, typename WarpExchangePairs::TempStorage, NullType>::Type WarpExchangePairsStorage;
-
-    typedef WarpExchange<OffsetT, ITEMS_PER_THREAD>                 WarpExchangeOffsets;
-    typedef WarpExchange<LengthT, ITEMS_PER_THREAD>                 WarpExchangeLengths;
-
-    typedef LengthOffsetPair WarpAggregates[WARPS];
-
-    // Shared memory type for this thread block
-    struct _TempStorage
-    {
-        // Aliasable storage layout
-        union Aliasable
-        {
-            struct
-            {
-                typename BlockDiscontinuityT::TempStorage       discontinuity;              // Smem needed for discontinuity detection
-                typename WarpScanPairs::TempStorage             warp_scan[WARPS];           // Smem needed for warp-synchronous scans
-                Uninitialized<LengthOffsetPair[WARPS]>          warp_aggregates;            // Smem needed for sharing warp-wide aggregates
-                typename TilePrefixCallbackOpT::TempStorage     prefix;                     // Smem needed for cooperative prefix callback
-            };
-
-            // Smem needed for input loading
-            typename BlockLoadT::TempStorage                    load;
-
-            // Aliasable layout needed for two-phase scatter
-            union ScatterAliasable
-            {
-                unsigned long long                              align;
-                WarpExchangePairsStorage                        exchange_pairs[ACTIVE_EXCHANGE_WARPS];
-                typename WarpExchangeOffsets::TempStorage       exchange_offsets[ACTIVE_EXCHANGE_WARPS];
-                typename WarpExchangeLengths::TempStorage       exchange_lengths[ACTIVE_EXCHANGE_WARPS];
-
-            } scatter_aliasable;
-
-        } aliasable;
-
-        OffsetT             tile_idx;                   // Shared tile index
-        LengthOffsetPair    tile_inclusive;             // Inclusive tile prefix
-        LengthOffsetPair    tile_exclusive;             // Exclusive tile prefix
-    };
-
-    // Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    //---------------------------------------------------------------------
-    // Per-thread fields
-    //---------------------------------------------------------------------
-
-    _TempStorage&                   temp_storage;       ///< Reference to temp_storage
-
-    WrappedInputIteratorT           d_in;               ///< Pointer to input sequence of data items
-    OffsetsOutputIteratorT          d_offsets_out;      ///< Input run offsets
-    LengthsOutputIteratorT          d_lengths_out;      ///< Output run lengths
-
-    EqualityOpT                     equality_op;        ///< T equality operator
-    ReduceBySegmentOpT              scan_op;            ///< Reduce-length-by-flag scan operator
-    OffsetT                         num_items;          ///< Total number of input items
-
-
-    //---------------------------------------------------------------------
-    // Constructor
-    //---------------------------------------------------------------------
-
-    // Constructor
-    __device__ __forceinline__
-    AgentRle(
-        TempStorage                 &temp_storage,      ///< [in] Reference to temp_storage
-        InputIteratorT              d_in,               ///< [in] Pointer to input sequence of data items
-        OffsetsOutputIteratorT      d_offsets_out,      ///< [out] Pointer to output sequence of run offsets
-        LengthsOutputIteratorT      d_lengths_out,      ///< [out] Pointer to output sequence of run lengths
-        EqualityOpT                 equality_op,        ///< [in] T equality operator
-        OffsetT                     num_items)          ///< [in] Total number of input items
-    :
-        temp_storage(temp_storage.Alias()),
-        d_in(d_in),
-        d_offsets_out(d_offsets_out),
-        d_lengths_out(d_lengths_out),
-        equality_op(equality_op),
-        scan_op(cub::Sum()),
-        num_items(num_items)
-    {}
-
-
-    //---------------------------------------------------------------------
-    // Utility methods for initializing the selections
-    //---------------------------------------------------------------------
-
-    template <bool FIRST_TILE, bool LAST_TILE>
-    __device__ __forceinline__ void InitializeSelections(
-        OffsetT             tile_offset,
-        OffsetT             num_remaining,
-        T                   (&items)[ITEMS_PER_THREAD],
-        LengthOffsetPair    (&lengths_and_num_runs)[ITEMS_PER_THREAD])
-    {
-        bool                head_flags[ITEMS_PER_THREAD];
-        bool                tail_flags[ITEMS_PER_THREAD];
-
-        OobInequalityOp<LAST_TILE> inequality_op(num_remaining, equality_op);
-
-        if (FIRST_TILE && LAST_TILE)
-        {
-            // First-and-last-tile always head-flags the first item and tail-flags the last item
-
-            BlockDiscontinuityT(temp_storage.aliasable.discontinuity).FlagHeadsAndTails(
-                head_flags, tail_flags, items, inequality_op);
-        }
-        else if (FIRST_TILE)
-        {
-            // First-tile always head-flags the first item
-
-            // Get the first item from the next tile
-            T tile_successor_item;
-            if (threadIdx.x == BLOCK_THREADS - 1)
-                tile_successor_item = d_in[tile_offset + TILE_ITEMS];
-
-            BlockDiscontinuityT(temp_storage.aliasable.discontinuity).FlagHeadsAndTails(
-                head_flags, tail_flags, tile_successor_item, items, inequality_op);
-        }
-        else if (LAST_TILE)
-        {
-            // Last-tile always flags the last item
-
-            // Get the last item from the previous tile
-            T tile_predecessor_item;
-            if (threadIdx.x == 0)
-                tile_predecessor_item = d_in[tile_offset - 1];
-
-            BlockDiscontinuityT(temp_storage.aliasable.discontinuity).FlagHeadsAndTails(
-                head_flags, tile_predecessor_item, tail_flags, items, inequality_op);
-        }
-        else
-        {
-            // Get the first item from the next tile
-            T tile_successor_item;
-            if (threadIdx.x == BLOCK_THREADS - 1)
-                tile_successor_item = d_in[tile_offset + TILE_ITEMS];
-
-            // Get the last item from the previous tile
-            T tile_predecessor_item;
-            if (threadIdx.x == 0)
-                tile_predecessor_item = d_in[tile_offset - 1];
-
-            BlockDiscontinuityT(temp_storage.aliasable.discontinuity).FlagHeadsAndTails(
-                head_flags, tile_predecessor_item, tail_flags, tile_successor_item, items, inequality_op);
-        }
-
-        // Zip counts and runs
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            lengths_and_num_runs[ITEM].key      = head_flags[ITEM] && (!tail_flags[ITEM]);
-            lengths_and_num_runs[ITEM].value    = ((!head_flags[ITEM]) || (!tail_flags[ITEM]));
-        }
-    }
-
-    //---------------------------------------------------------------------
-    // Scan utility methods
-    //---------------------------------------------------------------------
-
-    /**
-     * Scan of allocations
-     */
-    __device__ __forceinline__ void WarpScanAllocations(
-        LengthOffsetPair    &tile_aggregate,
-        LengthOffsetPair    &warp_aggregate,
-        LengthOffsetPair    &warp_exclusive_in_tile,
-        LengthOffsetPair    &thread_exclusive_in_warp,
-        LengthOffsetPair    (&lengths_and_num_runs)[ITEMS_PER_THREAD])
-    {
-        // Perform warpscans
-        unsigned int warp_id = ((WARPS == 1) ? 0 : threadIdx.x / WARP_THREADS);
-        int lane_id = LaneId();
-
-        LengthOffsetPair identity;
-        identity.key = 0;
-        identity.value = 0;
-
-        LengthOffsetPair thread_inclusive;
-        LengthOffsetPair thread_aggregate = internal::ThreadReduce(lengths_and_num_runs, scan_op);
-        WarpScanPairs(temp_storage.aliasable.warp_scan[warp_id]).Scan(
-            thread_aggregate,
-            thread_inclusive,
-            thread_exclusive_in_warp,
-            identity,
-            scan_op);
-
-        // Last lane in each warp shares its warp-aggregate
-        if (lane_id == WARP_THREADS - 1)
-            temp_storage.aliasable.warp_aggregates.Alias()[warp_id] = thread_inclusive;
-
-        CTA_SYNC();
-
-        // Accumulate total selected and the warp-wide prefix
-        warp_exclusive_in_tile          = identity;
-        warp_aggregate                  = temp_storage.aliasable.warp_aggregates.Alias()[warp_id];
-        tile_aggregate                  = temp_storage.aliasable.warp_aggregates.Alias()[0];
-
-        #pragma unroll
-        for (int WARP = 1; WARP < WARPS; ++WARP)
-        {
-            if (warp_id == WARP)
-                warp_exclusive_in_tile = tile_aggregate;
-
-            tile_aggregate = scan_op(tile_aggregate, temp_storage.aliasable.warp_aggregates.Alias()[WARP]);
-        }
-    }
-
-
-    //---------------------------------------------------------------------
-    // Utility methods for scattering selections
-    //---------------------------------------------------------------------
-
-    /**
-     * Two-phase scatter, specialized for warp time-slicing
-     */
-    template <bool FIRST_TILE>
-    __device__ __forceinline__ void ScatterTwoPhase(
-        OffsetT             tile_num_runs_exclusive_in_global,
-        OffsetT             warp_num_runs_aggregate,
-        OffsetT             warp_num_runs_exclusive_in_tile,
-        OffsetT             (&thread_num_runs_exclusive_in_warp)[ITEMS_PER_THREAD],
-        LengthOffsetPair    (&lengths_and_offsets)[ITEMS_PER_THREAD],
-        Int2Type<true>      is_warp_time_slice)
-    {
-        unsigned int warp_id = ((WARPS == 1) ? 0 : threadIdx.x / WARP_THREADS);
-        int lane_id = LaneId();
-
-        // Locally compact items within the warp (first warp)
-        if (warp_id == 0)
-        {
-            WarpExchangePairs(temp_storage.aliasable.scatter_aliasable.exchange_pairs[0]).ScatterToStriped(
-                lengths_and_offsets, thread_num_runs_exclusive_in_warp);
-        }
-
-        // Locally compact items within the warp (remaining warps)
-        #pragma unroll
-        for (int SLICE = 1; SLICE < WARPS; ++SLICE)
-        {
-            CTA_SYNC();
-
-            if (warp_id == SLICE)
-            {
-                WarpExchangePairs(temp_storage.aliasable.scatter_aliasable.exchange_pairs[0]).ScatterToStriped(
-                    lengths_and_offsets, thread_num_runs_exclusive_in_warp);
-            }
-        }
-
-        // Global scatter
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            if ((ITEM * WARP_THREADS) < warp_num_runs_aggregate - lane_id)
-            {
-                OffsetT item_offset =
-                    tile_num_runs_exclusive_in_global +
-                    warp_num_runs_exclusive_in_tile +
-                    (ITEM * WARP_THREADS) + lane_id;
-
-                // Scatter offset
-                d_offsets_out[item_offset] = lengths_and_offsets[ITEM].key;
-
-                // Scatter length if not the first (global) length
-                if ((!FIRST_TILE) || (ITEM != 0) || (threadIdx.x > 0))
-                {
-                    d_lengths_out[item_offset - 1] = lengths_and_offsets[ITEM].value;
-                }
-            }
-        }
-    }
-
-
-    /**
-     * Two-phase scatter
-     */
-    template <bool FIRST_TILE>
-    __device__ __forceinline__ void ScatterTwoPhase(
-        OffsetT             tile_num_runs_exclusive_in_global,
-        OffsetT             warp_num_runs_aggregate,
-        OffsetT             warp_num_runs_exclusive_in_tile,
-        OffsetT             (&thread_num_runs_exclusive_in_warp)[ITEMS_PER_THREAD],
-        LengthOffsetPair    (&lengths_and_offsets)[ITEMS_PER_THREAD],
-        Int2Type<false>     is_warp_time_slice)
-    {
-        unsigned int warp_id = ((WARPS == 1) ? 0 : threadIdx.x / WARP_THREADS);
-        int lane_id = LaneId();
-
-        // Unzip
-        OffsetT run_offsets[ITEMS_PER_THREAD];
-        LengthT run_lengths[ITEMS_PER_THREAD];
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            run_offsets[ITEM] = lengths_and_offsets[ITEM].key;
-            run_lengths[ITEM] = lengths_and_offsets[ITEM].value;
-        }
-
-        WarpExchangeOffsets(temp_storage.aliasable.scatter_aliasable.exchange_offsets[warp_id]).ScatterToStriped(
-            run_offsets, thread_num_runs_exclusive_in_warp);
-
-        WARP_SYNC(0xffffffff);
-
-        WarpExchangeLengths(temp_storage.aliasable.scatter_aliasable.exchange_lengths[warp_id]).ScatterToStriped(
-            run_lengths, thread_num_runs_exclusive_in_warp);
-
-        // Global scatter
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            if ((ITEM * WARP_THREADS) + lane_id < warp_num_runs_aggregate)
-            {
-                OffsetT item_offset =
-                    tile_num_runs_exclusive_in_global +
-                    warp_num_runs_exclusive_in_tile +
-                    (ITEM * WARP_THREADS) + lane_id;
-
-                // Scatter offset
-                d_offsets_out[item_offset] = run_offsets[ITEM];
-
-                // Scatter length if not the first (global) length
-                if ((!FIRST_TILE) || (ITEM != 0) || (threadIdx.x > 0))
-                {
-                    d_lengths_out[item_offset - 1] = run_lengths[ITEM];
-                }
-            }
-        }
-    }
-
-
-    /**
-     * Direct scatter
-     */
-    template <bool FIRST_TILE>
-    __device__ __forceinline__ void ScatterDirect(
-        OffsetT             tile_num_runs_exclusive_in_global,
-        OffsetT             warp_num_runs_aggregate,
-        OffsetT             warp_num_runs_exclusive_in_tile,
-        OffsetT             (&thread_num_runs_exclusive_in_warp)[ITEMS_PER_THREAD],
-        LengthOffsetPair    (&lengths_and_offsets)[ITEMS_PER_THREAD])
-    {
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            if (thread_num_runs_exclusive_in_warp[ITEM] < warp_num_runs_aggregate)
-            {
-                OffsetT item_offset =
-                    tile_num_runs_exclusive_in_global +
-                    warp_num_runs_exclusive_in_tile +
-                    thread_num_runs_exclusive_in_warp[ITEM];
-
-                // Scatter offset
-                d_offsets_out[item_offset] = lengths_and_offsets[ITEM].key;
-
-                // Scatter length if not the first (global) length
-                if (item_offset >= 1)
-                {
-                    d_lengths_out[item_offset - 1] = lengths_and_offsets[ITEM].value;
-                }
-            }
-        }
-    }
-
-
-    /**
-     * Scatter
-     */
-    template <bool FIRST_TILE>
-    __device__ __forceinline__ void Scatter(
-        OffsetT             tile_num_runs_aggregate,
-        OffsetT             tile_num_runs_exclusive_in_global,
-        OffsetT             warp_num_runs_aggregate,
-        OffsetT             warp_num_runs_exclusive_in_tile,
-        OffsetT             (&thread_num_runs_exclusive_in_warp)[ITEMS_PER_THREAD],
-        LengthOffsetPair    (&lengths_and_offsets)[ITEMS_PER_THREAD])
-    {
-        if ((ITEMS_PER_THREAD == 1) || (tile_num_runs_aggregate < BLOCK_THREADS))
-        {
-            // Direct scatter if the warp has any items
-            if (warp_num_runs_aggregate)
-            {
-                ScatterDirect<FIRST_TILE>(
-                    tile_num_runs_exclusive_in_global,
-                    warp_num_runs_aggregate,
-                    warp_num_runs_exclusive_in_tile,
-                    thread_num_runs_exclusive_in_warp,
-                    lengths_and_offsets);
-            }
-        }
-        else
-        {
-            // Scatter two phase
-            ScatterTwoPhase<FIRST_TILE>(
-                tile_num_runs_exclusive_in_global,
-                warp_num_runs_aggregate,
-                warp_num_runs_exclusive_in_tile,
-                thread_num_runs_exclusive_in_warp,
-                lengths_and_offsets,
-                Int2Type<STORE_WARP_TIME_SLICING>());
-        }
-    }
-
-
-
-    //---------------------------------------------------------------------
-    // Cooperatively scan a device-wide sequence of tiles with other CTAs
-    //---------------------------------------------------------------------
-
-    /**
-     * Process a tile of input (dynamic chained scan)
-     */
-    template <
-        bool                LAST_TILE>
-    __device__ __forceinline__ LengthOffsetPair ConsumeTile(
-        OffsetT             num_items,          ///< Total number of global input items
-        OffsetT             num_remaining,      ///< Number of global input items remaining (including this tile)
-        int                 tile_idx,           ///< Tile index
-        OffsetT             tile_offset,        ///< Tile offset
-        ScanTileStateT      &tile_status)       ///< Global list of tile status
-    {
-        if (tile_idx == 0)
-        {
-            // First tile
-
-            // Load items
-            T items[ITEMS_PER_THREAD];
-            if (LAST_TILE)
-                BlockLoadT(temp_storage.aliasable.load).Load(d_in + tile_offset, items, num_remaining, T());
-            else
-                BlockLoadT(temp_storage.aliasable.load).Load(d_in + tile_offset, items);
-
-            if (SYNC_AFTER_LOAD)
-                CTA_SYNC();
-
-            // Set flags
-            LengthOffsetPair    lengths_and_num_runs[ITEMS_PER_THREAD];
-
-            InitializeSelections<true, LAST_TILE>(
-                tile_offset,
-                num_remaining,
-                items,
-                lengths_and_num_runs);
-
-            // Exclusive scan of lengths and runs
-            LengthOffsetPair tile_aggregate;
-            LengthOffsetPair warp_aggregate;
-            LengthOffsetPair warp_exclusive_in_tile;
-            LengthOffsetPair thread_exclusive_in_warp;
-
-            WarpScanAllocations(
-                tile_aggregate,
-                warp_aggregate,
-                warp_exclusive_in_tile,
-                thread_exclusive_in_warp,
-                lengths_and_num_runs);
-
-            // Update tile status if this is not the last tile
-            if (!LAST_TILE && (threadIdx.x == 0))
-                tile_status.SetInclusive(0, tile_aggregate);
-
-            // Update thread_exclusive_in_warp to fold in warp run-length
-            if (thread_exclusive_in_warp.key == 0)
-                thread_exclusive_in_warp.value += warp_exclusive_in_tile.value;
-
-            LengthOffsetPair    lengths_and_offsets[ITEMS_PER_THREAD];
-            OffsetT             thread_num_runs_exclusive_in_warp[ITEMS_PER_THREAD];
-            LengthOffsetPair    lengths_and_num_runs2[ITEMS_PER_THREAD];
-
-            // Downsweep scan through lengths_and_num_runs
-            internal::ThreadScanExclusive(lengths_and_num_runs, lengths_and_num_runs2, scan_op, thread_exclusive_in_warp);
-
-            // Zip
-
-            #pragma unroll
-            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-            {
-                lengths_and_offsets[ITEM].value         = lengths_and_num_runs2[ITEM].value;
-                lengths_and_offsets[ITEM].key        = tile_offset + (threadIdx.x * ITEMS_PER_THREAD) + ITEM;
-                thread_num_runs_exclusive_in_warp[ITEM] = (lengths_and_num_runs[ITEM].key) ?
-                                                                lengths_and_num_runs2[ITEM].key :         // keep
-                                                                WARP_THREADS * ITEMS_PER_THREAD;            // discard
-            }
-
-            OffsetT tile_num_runs_aggregate              = tile_aggregate.key;
-            OffsetT tile_num_runs_exclusive_in_global    = 0;
-            OffsetT warp_num_runs_aggregate              = warp_aggregate.key;
-            OffsetT warp_num_runs_exclusive_in_tile      = warp_exclusive_in_tile.key;
-
-            // Scatter
-            Scatter<true>(
-                tile_num_runs_aggregate,
-                tile_num_runs_exclusive_in_global,
-                warp_num_runs_aggregate,
-                warp_num_runs_exclusive_in_tile,
-                thread_num_runs_exclusive_in_warp,
-                lengths_and_offsets);
-
-            // Return running total (inclusive of this tile)
-            return tile_aggregate;
-        }
-        else
-        {
-            // Not first tile
-
-            // Load items
-            T items[ITEMS_PER_THREAD];
-            if (LAST_TILE)
-                BlockLoadT(temp_storage.aliasable.load).Load(d_in + tile_offset, items, num_remaining, T());
-            else
-                BlockLoadT(temp_storage.aliasable.load).Load(d_in + tile_offset, items);
-
-            if (SYNC_AFTER_LOAD)
-                CTA_SYNC();
-
-            // Set flags
-            LengthOffsetPair    lengths_and_num_runs[ITEMS_PER_THREAD];
-
-            InitializeSelections<false, LAST_TILE>(
-                tile_offset,
-                num_remaining,
-                items,
-                lengths_and_num_runs);
-
-            // Exclusive scan of lengths and runs
-            LengthOffsetPair tile_aggregate;
-            LengthOffsetPair warp_aggregate;
-            LengthOffsetPair warp_exclusive_in_tile;
-            LengthOffsetPair thread_exclusive_in_warp;
-
-            WarpScanAllocations(
-                tile_aggregate,
-                warp_aggregate,
-                warp_exclusive_in_tile,
-                thread_exclusive_in_warp,
-                lengths_and_num_runs);
-
-            // First warp computes tile prefix in lane 0
-            TilePrefixCallbackOpT prefix_op(tile_status, temp_storage.aliasable.prefix, Sum(), tile_idx);
-            unsigned int warp_id = ((WARPS == 1) ? 0 : threadIdx.x / WARP_THREADS);
-            if (warp_id == 0)
-            {
-                prefix_op(tile_aggregate);
-                if (threadIdx.x == 0)
-                    temp_storage.tile_exclusive = prefix_op.exclusive_prefix;
-            }
-
-            CTA_SYNC();
-
-            LengthOffsetPair tile_exclusive_in_global = temp_storage.tile_exclusive;
-
-            // Update thread_exclusive_in_warp to fold in warp and tile run-lengths
-            LengthOffsetPair thread_exclusive = scan_op(tile_exclusive_in_global, warp_exclusive_in_tile);
-            if (thread_exclusive_in_warp.key == 0)
-                thread_exclusive_in_warp.value += thread_exclusive.value;
-
-            // Downsweep scan through lengths_and_num_runs
-            LengthOffsetPair    lengths_and_num_runs2[ITEMS_PER_THREAD];
-            LengthOffsetPair    lengths_and_offsets[ITEMS_PER_THREAD];
-            OffsetT             thread_num_runs_exclusive_in_warp[ITEMS_PER_THREAD];
-
-            internal::ThreadScanExclusive(lengths_and_num_runs, lengths_and_num_runs2, scan_op, thread_exclusive_in_warp);
-
-            // Zip
-            #pragma unroll
-            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-            {
-                lengths_and_offsets[ITEM].value         = lengths_and_num_runs2[ITEM].value;
-                lengths_and_offsets[ITEM].key        = tile_offset + (threadIdx.x * ITEMS_PER_THREAD) + ITEM;
-                thread_num_runs_exclusive_in_warp[ITEM] = (lengths_and_num_runs[ITEM].key) ?
-                                                                lengths_and_num_runs2[ITEM].key :         // keep
-                                                                WARP_THREADS * ITEMS_PER_THREAD;            // discard
-            }
-
-            OffsetT tile_num_runs_aggregate              = tile_aggregate.key;
-            OffsetT tile_num_runs_exclusive_in_global    = tile_exclusive_in_global.key;
-            OffsetT warp_num_runs_aggregate              = warp_aggregate.key;
-            OffsetT warp_num_runs_exclusive_in_tile      = warp_exclusive_in_tile.key;
-
-            // Scatter
-            Scatter<false>(
-                tile_num_runs_aggregate,
-                tile_num_runs_exclusive_in_global,
-                warp_num_runs_aggregate,
-                warp_num_runs_exclusive_in_tile,
-                thread_num_runs_exclusive_in_warp,
-                lengths_and_offsets);
-
-            // Return running total (inclusive of this tile)
-            return prefix_op.inclusive_prefix;
-        }
-    }
-
-
-    /**
-     * Scan tiles of items as part of a dynamic chained scan
-     */
-    template <typename NumRunsIteratorT>            ///< Output iterator type for recording number of items selected
-    __device__ __forceinline__ void ConsumeRange(
-        int                 num_tiles,              ///< Total number of input tiles
-        ScanTileStateT&     tile_status,            ///< Global list of tile status
-        NumRunsIteratorT    d_num_runs_out)         ///< Output pointer for total number of runs identified
-    {
-        // Blocks are launched in increasing order, so just assign one tile per block
-        int     tile_idx        = (blockIdx.x * gridDim.y) + blockIdx.y;    // Current tile index
-        OffsetT tile_offset     = tile_idx * TILE_ITEMS;                  // Global offset for the current tile
-        OffsetT num_remaining   = num_items - tile_offset;                  // Remaining items (including this tile)
-
-        if (tile_idx < num_tiles - 1)
-        {
-            // Not the last tile (full)
-            ConsumeTile<false>(num_items, num_remaining, tile_idx, tile_offset, tile_status);
-        }
-        else if (num_remaining > 0)
-        {
-            // The last tile (possibly partially-full)
-            LengthOffsetPair running_total = ConsumeTile<true>(num_items, num_remaining, tile_idx, tile_offset, tile_status);
-
-            if (threadIdx.x == 0)
-            {
-                // Output the total number of items selected
-                *d_num_runs_out = running_total.key;
-
-                // The inclusive prefix contains accumulated length reduction for the last run
-                if (running_total.key > 0)
-                    d_lengths_out[running_total.key - 1] = running_total.value;
-            }
-        }
-    }
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/agent/agent_scan.cuh b/pfsimulator/third_party/cub/agent/agent_scan.cuh
deleted file mode 100644
index 9368615ef..000000000
--- a/pfsimulator/third_party/cub/agent/agent_scan.cuh
+++ /dev/null
@@ -1,471 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::AgentScan implements a stateful abstraction of CUDA thread blocks for participating in device-wide prefix scan .
- */
-
-#pragma once
-
-#include <iterator>
-
-#include "single_pass_scan_operators.cuh"
-#include "../block/block_load.cuh"
-#include "../block/block_store.cuh"
-#include "../block/block_scan.cuh"
-#include "../grid/grid_queue.cuh"
-#include "../iterator/cache_modified_input_iterator.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/******************************************************************************
- * Tuning policy types
- ******************************************************************************/
-
-/**
- * Parameterizable tuning policy type for AgentScan
- */
-template <
-    int                         _BLOCK_THREADS,                 ///< Threads per thread block
-    int                         _ITEMS_PER_THREAD,              ///< Items per thread (per tile of input)
-    BlockLoadAlgorithm          _LOAD_ALGORITHM,                ///< The BlockLoad algorithm to use
-    CacheLoadModifier           _LOAD_MODIFIER,                 ///< Cache load modifier for reading input elements
-    BlockStoreAlgorithm         _STORE_ALGORITHM,               ///< The BlockStore algorithm to use
-    BlockScanAlgorithm          _SCAN_ALGORITHM>                ///< The BlockScan algorithm to use
-struct AgentScanPolicy
-{
-    enum
-    {
-        BLOCK_THREADS           = _BLOCK_THREADS,               ///< Threads per thread block
-        ITEMS_PER_THREAD        = _ITEMS_PER_THREAD,            ///< Items per thread (per tile of input)
-    };
-
-    static const BlockLoadAlgorithm     LOAD_ALGORITHM          = _LOAD_ALGORITHM;          ///< The BlockLoad algorithm to use
-    static const CacheLoadModifier      LOAD_MODIFIER           = _LOAD_MODIFIER;           ///< Cache load modifier for reading input elements
-    static const BlockStoreAlgorithm    STORE_ALGORITHM         = _STORE_ALGORITHM;         ///< The BlockStore algorithm to use
-    static const BlockScanAlgorithm     SCAN_ALGORITHM          = _SCAN_ALGORITHM;          ///< The BlockScan algorithm to use
-};
-
-
-
-
-/******************************************************************************
- * Thread block abstractions
- ******************************************************************************/
-
-/**
- * \brief AgentScan implements a stateful abstraction of CUDA thread blocks for participating in device-wide prefix scan .
- */
-template <
-    typename AgentScanPolicyT,      ///< Parameterized AgentScanPolicyT tuning policy type
-    typename InputIteratorT,        ///< Random-access input iterator type
-    typename OutputIteratorT,       ///< Random-access output iterator type
-    typename ScanOpT,               ///< Scan functor type
-    typename InitValueT,            ///< The init_value element for ScanOpT type (cub::NullType for inclusive scan)
-    typename OffsetT>               ///< Signed integer type for global offsets
-struct AgentScan
-{
-    //---------------------------------------------------------------------
-    // Types and constants
-    //---------------------------------------------------------------------
-
-    // The input value type
-    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
-
-    // The output value type
-    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
-        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
-        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
-
-    // Tile status descriptor interface type
-    typedef ScanTileState<OutputT> ScanTileStateT;
-
-    // Input iterator wrapper type (for applying cache modifier)
-    typedef typename If<IsPointer<InputIteratorT>::VALUE,
-            CacheModifiedInputIterator<AgentScanPolicyT::LOAD_MODIFIER, InputT, OffsetT>,   // Wrap the native input pointer with CacheModifiedInputIterator
-            InputIteratorT>::Type                                                           // Directly use the supplied input iterator type
-        WrappedInputIteratorT;
-
-    // Constants
-    enum
-    {
-        IS_INCLUSIVE        = Equals<InitValueT, NullType>::VALUE,            // Inclusive scan if no init_value type is provided
-        BLOCK_THREADS       = AgentScanPolicyT::BLOCK_THREADS,
-        ITEMS_PER_THREAD    = AgentScanPolicyT::ITEMS_PER_THREAD,
-        TILE_ITEMS          = BLOCK_THREADS * ITEMS_PER_THREAD,
-    };
-
-    // Parameterized BlockLoad type
-    typedef BlockLoad<
-            OutputT,
-            AgentScanPolicyT::BLOCK_THREADS,
-            AgentScanPolicyT::ITEMS_PER_THREAD,
-            AgentScanPolicyT::LOAD_ALGORITHM>
-        BlockLoadT;
-
-    // Parameterized BlockStore type
-    typedef BlockStore<
-            OutputT,
-            AgentScanPolicyT::BLOCK_THREADS,
-            AgentScanPolicyT::ITEMS_PER_THREAD,
-            AgentScanPolicyT::STORE_ALGORITHM>
-        BlockStoreT;
-
-    // Parameterized BlockScan type
-    typedef BlockScan<
-            OutputT,
-            AgentScanPolicyT::BLOCK_THREADS,
-            AgentScanPolicyT::SCAN_ALGORITHM>
-        BlockScanT;
-
-    // Callback type for obtaining tile prefix during block scan
-    typedef TilePrefixCallbackOp<
-            OutputT,
-            ScanOpT,
-            ScanTileStateT>
-        TilePrefixCallbackOpT;
-
-    // Stateful BlockScan prefix callback type for managing a running total while scanning consecutive tiles
-    typedef BlockScanRunningPrefixOp<
-            OutputT,
-            ScanOpT>
-        RunningPrefixCallbackOp;
-
-    // Shared memory type for this thread block
-    union _TempStorage
-    {
-        typename BlockLoadT::TempStorage    load;       // Smem needed for tile loading
-        typename BlockStoreT::TempStorage   store;      // Smem needed for tile storing
-
-        struct
-        {
-            typename TilePrefixCallbackOpT::TempStorage  prefix;     // Smem needed for cooperative prefix callback
-            typename BlockScanT::TempStorage             scan;       // Smem needed for tile scanning
-        };
-    };
-
-    // Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    //---------------------------------------------------------------------
-    // Per-thread fields
-    //---------------------------------------------------------------------
-
-    _TempStorage&               temp_storage;       ///< Reference to temp_storage
-    WrappedInputIteratorT       d_in;               ///< Input data
-    OutputIteratorT             d_out;              ///< Output data
-    ScanOpT                     scan_op;            ///< Binary scan operator
-    InitValueT                  init_value;         ///< The init_value element for ScanOpT
-
-
-    //---------------------------------------------------------------------
-    // Block scan utility methods
-    //---------------------------------------------------------------------
-
-    /**
-     * Exclusive scan specialization (first tile)
-     */
-    __device__ __forceinline__
-    void ScanTile(
-        OutputT             (&items)[ITEMS_PER_THREAD],
-        OutputT             init_value,
-        ScanOpT             scan_op,
-        OutputT             &block_aggregate,
-        Int2Type<false>     /*is_inclusive*/)
-    {
-        BlockScanT(temp_storage.scan).ExclusiveScan(items, items, init_value, scan_op, block_aggregate);
-        block_aggregate = scan_op(init_value, block_aggregate);
-    }
-
-
-    /**
-     * Inclusive scan specialization (first tile)
-     */
-    __device__ __forceinline__
-    void ScanTile(
-        OutputT             (&items)[ITEMS_PER_THREAD],
-        InitValueT          /*init_value*/,
-        ScanOpT             scan_op,
-        OutputT             &block_aggregate,
-        Int2Type<true>      /*is_inclusive*/)
-    {
-        BlockScanT(temp_storage.scan).InclusiveScan(items, items, scan_op, block_aggregate);
-    }
-
-
-    /**
-     * Exclusive scan specialization (subsequent tiles)
-     */
-    template <typename PrefixCallback>
-    __device__ __forceinline__
-    void ScanTile(
-        OutputT             (&items)[ITEMS_PER_THREAD],
-        ScanOpT             scan_op,
-        PrefixCallback      &prefix_op,
-        Int2Type<false>     /*is_inclusive*/)
-    {
-        BlockScanT(temp_storage.scan).ExclusiveScan(items, items, scan_op, prefix_op);
-    }
-
-
-    /**
-     * Inclusive scan specialization (subsequent tiles)
-     */
-    template <typename PrefixCallback>
-    __device__ __forceinline__
-    void ScanTile(
-        OutputT             (&items)[ITEMS_PER_THREAD],
-        ScanOpT             scan_op,
-        PrefixCallback      &prefix_op,
-        Int2Type<true>      /*is_inclusive*/)
-    {
-        BlockScanT(temp_storage.scan).InclusiveScan(items, items, scan_op, prefix_op);
-    }
-
-
-    //---------------------------------------------------------------------
-    // Constructor
-    //---------------------------------------------------------------------
-
-    // Constructor
-    __device__ __forceinline__
-    AgentScan(
-        TempStorage&    temp_storage,       ///< Reference to temp_storage
-        InputIteratorT  d_in,               ///< Input data
-        OutputIteratorT d_out,              ///< Output data
-        ScanOpT         scan_op,            ///< Binary scan operator
-        InitValueT      init_value)         ///< Initial value to seed the exclusive scan
-    :
-        temp_storage(temp_storage.Alias()),
-        d_in(d_in),
-        d_out(d_out),
-        scan_op(scan_op),
-        init_value(init_value)
-    {}
-
-
-    //---------------------------------------------------------------------
-    // Cooperatively scan a device-wide sequence of tiles with other CTAs
-    //---------------------------------------------------------------------
-
-    /**
-     * Process a tile of input (dynamic chained scan)
-     */
-    template <bool IS_LAST_TILE>                ///< Whether the current tile is the last tile
-    __device__ __forceinline__ void ConsumeTile(
-        OffsetT             num_remaining,      ///< Number of global input items remaining (including this tile)
-        int                 tile_idx,           ///< Tile index
-        OffsetT             tile_offset,        ///< Tile offset
-        ScanTileStateT&     tile_state)         ///< Global tile state descriptor
-    {
-        // Load items
-        OutputT items[ITEMS_PER_THREAD];
-
-        if (IS_LAST_TILE)
-            BlockLoadT(temp_storage.load).Load(d_in + tile_offset, items, num_remaining);
-        else
-            BlockLoadT(temp_storage.load).Load(d_in + tile_offset, items);
-
-        CTA_SYNC();
-
-        // Perform tile scan
-        if (tile_idx == 0)
-        {
-            // Scan first tile
-            OutputT block_aggregate;
-            ScanTile(items, init_value, scan_op, block_aggregate, Int2Type<IS_INCLUSIVE>());
-            if ((!IS_LAST_TILE) && (threadIdx.x == 0))
-                tile_state.SetInclusive(0, block_aggregate);
-        }
-        else
-        {
-            // Scan non-first tile
-            TilePrefixCallbackOpT prefix_op(tile_state, temp_storage.prefix, scan_op, tile_idx);
-            ScanTile(items, scan_op, prefix_op, Int2Type<IS_INCLUSIVE>());
-        }
-
-        CTA_SYNC();
-
-        // Store items
-        if (IS_LAST_TILE)
-            BlockStoreT(temp_storage.store).Store(d_out + tile_offset, items, num_remaining);
-        else
-            BlockStoreT(temp_storage.store).Store(d_out + tile_offset, items);
-    }
-
-
-    /**
-     * Scan tiles of items as part of a dynamic chained scan
-     */
-    __device__ __forceinline__ void ConsumeRange(
-        int                 num_items,          ///< Total number of input items
-        ScanTileStateT&     tile_state,         ///< Global tile state descriptor
-        int                 start_tile)         ///< The starting tile for the current grid
-    {
-        // Blocks are launched in increasing order, so just assign one tile per block
-        int     tile_idx        = start_tile + blockIdx.x;          // Current tile index
-        OffsetT tile_offset     = OffsetT(TILE_ITEMS) * tile_idx;   // Global offset for the current tile
-        OffsetT num_remaining   = num_items - tile_offset;          // Remaining items (including this tile)
-
-        if (num_remaining > TILE_ITEMS)
-        {
-            // Not last tile
-            ConsumeTile<false>(num_remaining, tile_idx, tile_offset, tile_state);
-        }
-        else if (num_remaining > 0)
-        {
-            // Last tile
-            ConsumeTile<true>(num_remaining, tile_idx, tile_offset, tile_state);
-        }
-    }
-
-
-    //---------------------------------------------------------------------
-    // Scan an sequence of consecutive tiles (independent of other thread blocks)
-    //---------------------------------------------------------------------
-
-    /**
-     * Process a tile of input
-     */
-    template <
-        bool                        IS_FIRST_TILE,
-        bool                        IS_LAST_TILE>
-    __device__ __forceinline__ void ConsumeTile(
-        OffsetT                     tile_offset,                ///< Tile offset
-        RunningPrefixCallbackOp&    prefix_op,                  ///< Running prefix operator
-        int                         valid_items = TILE_ITEMS)   ///< Number of valid items in the tile
-    {
-        // Load items
-        OutputT items[ITEMS_PER_THREAD];
-
-        if (IS_LAST_TILE)
-            BlockLoadT(temp_storage.load).Load(d_in + tile_offset, items, valid_items);
-        else
-            BlockLoadT(temp_storage.load).Load(d_in + tile_offset, items);
-
-        CTA_SYNC();
-
-        // Block scan
-        if (IS_FIRST_TILE)
-        {
-            OutputT block_aggregate;
-            ScanTile(items, init_value, scan_op, block_aggregate, Int2Type<IS_INCLUSIVE>());
-            prefix_op.running_total = block_aggregate;
-        }
-        else
-        {
-            ScanTile(items, scan_op, prefix_op, Int2Type<IS_INCLUSIVE>());
-        }
-
-        CTA_SYNC();
-
-        // Store items
-        if (IS_LAST_TILE)
-            BlockStoreT(temp_storage.store).Store(d_out + tile_offset, items, valid_items);
-        else
-            BlockStoreT(temp_storage.store).Store(d_out + tile_offset, items);
-    }
-
-
-    /**
-     * Scan a consecutive share of input tiles
-     */
-    __device__ __forceinline__ void ConsumeRange(
-        OffsetT  range_offset,      ///< [in] Threadblock begin offset (inclusive)
-        OffsetT  range_end)         ///< [in] Threadblock end offset (exclusive)
-    {
-        BlockScanRunningPrefixOp<OutputT, ScanOpT> prefix_op(scan_op);
-
-        if (range_offset + TILE_ITEMS <= range_end)
-        {
-            // Consume first tile of input (full)
-            ConsumeTile<true, true>(range_offset, prefix_op);
-            range_offset += TILE_ITEMS;
-
-            // Consume subsequent full tiles of input
-            while (range_offset + TILE_ITEMS <= range_end)
-            {
-                ConsumeTile<false, true>(range_offset, prefix_op);
-                range_offset += TILE_ITEMS;
-            }
-
-            // Consume a partially-full tile
-            if (range_offset < range_end)
-            {
-                int valid_items = range_end - range_offset;
-                ConsumeTile<false, false>(range_offset, prefix_op, valid_items);
-            }
-        }
-        else
-        {
-            // Consume the first tile of input (partially-full)
-            int valid_items = range_end - range_offset;
-            ConsumeTile<true, false>(range_offset, prefix_op, valid_items);
-        }
-    }
-
-
-    /**
-     * Scan a consecutive share of input tiles, seeded with the specified prefix value
-     */
-    __device__ __forceinline__ void ConsumeRange(
-        OffsetT range_offset,                       ///< [in] Threadblock begin offset (inclusive)
-        OffsetT range_end,                          ///< [in] Threadblock end offset (exclusive)
-        OutputT prefix)                             ///< [in] The prefix to apply to the scan segment
-    {
-        BlockScanRunningPrefixOp<OutputT, ScanOpT> prefix_op(prefix, scan_op);
-
-        // Consume full tiles of input
-        while (range_offset + TILE_ITEMS <= range_end)
-        {
-            ConsumeTile<true, false>(range_offset, prefix_op);
-            range_offset += TILE_ITEMS;
-        }
-
-        // Consume a partially-full tile
-        if (range_offset < range_end)
-        {
-            int valid_items = range_end - range_offset;
-            ConsumeTile<false, false>(range_offset, prefix_op, valid_items);
-        }
-    }
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/agent/agent_segment_fixup.cuh b/pfsimulator/third_party/cub/agent/agent_segment_fixup.cuh
deleted file mode 100644
index e2de58ed6..000000000
--- a/pfsimulator/third_party/cub/agent/agent_segment_fixup.cuh
+++ /dev/null
@@ -1,375 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::AgentSegmentFixup implements a stateful abstraction of CUDA thread blocks for participating in device-wide reduce-value-by-key.
- */
-
-#pragma once
-
-#include <iterator>
-
-#include "single_pass_scan_operators.cuh"
-#include "../block/block_load.cuh"
-#include "../block/block_store.cuh"
-#include "../block/block_scan.cuh"
-#include "../block/block_discontinuity.cuh"
-#include "../iterator/cache_modified_input_iterator.cuh"
-#include "../iterator/constant_input_iterator.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/******************************************************************************
- * Tuning policy types
- ******************************************************************************/
-
-/**
- * Parameterizable tuning policy type for AgentSegmentFixup
- */
-template <
-    int                         _BLOCK_THREADS,                 ///< Threads per thread block
-    int                         _ITEMS_PER_THREAD,              ///< Items per thread (per tile of input)
-    BlockLoadAlgorithm          _LOAD_ALGORITHM,                ///< The BlockLoad algorithm to use
-    CacheLoadModifier           _LOAD_MODIFIER,                 ///< Cache load modifier for reading input elements
-    BlockScanAlgorithm          _SCAN_ALGORITHM>                ///< The BlockScan algorithm to use
-struct AgentSegmentFixupPolicy
-{
-    enum
-    {
-        BLOCK_THREADS           = _BLOCK_THREADS,               ///< Threads per thread block
-        ITEMS_PER_THREAD        = _ITEMS_PER_THREAD,            ///< Items per thread (per tile of input)
-    };
-
-    static const BlockLoadAlgorithm     LOAD_ALGORITHM          = _LOAD_ALGORITHM;      ///< The BlockLoad algorithm to use
-    static const CacheLoadModifier      LOAD_MODIFIER           = _LOAD_MODIFIER;       ///< Cache load modifier for reading input elements
-    static const BlockScanAlgorithm     SCAN_ALGORITHM          = _SCAN_ALGORITHM;      ///< The BlockScan algorithm to use
-};
-
-
-/******************************************************************************
- * Thread block abstractions
- ******************************************************************************/
-
-/**
- * \brief AgentSegmentFixup implements a stateful abstraction of CUDA thread blocks for participating in device-wide reduce-value-by-key
- */
-template <
-    typename    AgentSegmentFixupPolicyT,       ///< Parameterized AgentSegmentFixupPolicy tuning policy type
-    typename    PairsInputIteratorT,            ///< Random-access input iterator type for keys
-    typename    AggregatesOutputIteratorT,      ///< Random-access output iterator type for values
-    typename    EqualityOpT,                    ///< KeyT equality operator type
-    typename    ReductionOpT,                   ///< ValueT reduction operator type
-    typename    OffsetT>                        ///< Signed integer type for global offsets
-struct AgentSegmentFixup
-{
-    //---------------------------------------------------------------------
-    // Types and constants
-    //---------------------------------------------------------------------
-
-    // Data type of key-value input iterator
-    typedef typename std::iterator_traits<PairsInputIteratorT>::value_type KeyValuePairT;
-
-    // Value type
-    typedef typename KeyValuePairT::Value ValueT;
-
-    // Tile status descriptor interface type
-    typedef ReduceByKeyScanTileState<ValueT, OffsetT> ScanTileStateT;
-
-    // Constants
-    enum
-    {
-        BLOCK_THREADS       = AgentSegmentFixupPolicyT::BLOCK_THREADS,
-        ITEMS_PER_THREAD    = AgentSegmentFixupPolicyT::ITEMS_PER_THREAD,
-        TILE_ITEMS          = BLOCK_THREADS * ITEMS_PER_THREAD,
-
-        // Whether or not do fixup using RLE + global atomics
-        USE_ATOMIC_FIXUP    = (CUB_PTX_ARCH >= 350) && 
-                                (Equals<ValueT, float>::VALUE || 
-                                 Equals<ValueT, int>::VALUE ||
-                                 Equals<ValueT, unsigned int>::VALUE ||
-                                 Equals<ValueT, unsigned long long>::VALUE),
-
-        // Whether or not the scan operation has a zero-valued identity value (true if we're performing addition on a primitive type)
-        HAS_IDENTITY_ZERO   = (Equals<ReductionOpT, cub::Sum>::VALUE) && (Traits<ValueT>::PRIMITIVE),
-    };
-
-    // Cache-modified Input iterator wrapper type (for applying cache modifier) for keys
-    typedef typename If<IsPointer<PairsInputIteratorT>::VALUE,
-            CacheModifiedInputIterator<AgentSegmentFixupPolicyT::LOAD_MODIFIER, KeyValuePairT, OffsetT>,    // Wrap the native input pointer with CacheModifiedValuesInputIterator
-            PairsInputIteratorT>::Type                                                                      // Directly use the supplied input iterator type
-        WrappedPairsInputIteratorT;
-
-    // Cache-modified Input iterator wrapper type (for applying cache modifier) for fixup values
-    typedef typename If<IsPointer<AggregatesOutputIteratorT>::VALUE,
-            CacheModifiedInputIterator<AgentSegmentFixupPolicyT::LOAD_MODIFIER, ValueT, OffsetT>,    // Wrap the native input pointer with CacheModifiedValuesInputIterator
-            AggregatesOutputIteratorT>::Type                                                        // Directly use the supplied input iterator type
-        WrappedFixupInputIteratorT;
-
-    // Reduce-value-by-segment scan operator
-    typedef ReduceByKeyOp<cub::Sum> ReduceBySegmentOpT;
-
-    // Parameterized BlockLoad type for pairs
-    typedef BlockLoad<
-            KeyValuePairT,
-            BLOCK_THREADS,
-            ITEMS_PER_THREAD,
-            AgentSegmentFixupPolicyT::LOAD_ALGORITHM>
-        BlockLoadPairs;
-
-    // Parameterized BlockScan type
-    typedef BlockScan<
-            KeyValuePairT,
-            BLOCK_THREADS,
-            AgentSegmentFixupPolicyT::SCAN_ALGORITHM>
-        BlockScanT;
-
-    // Callback type for obtaining tile prefix during block scan
-    typedef TilePrefixCallbackOp<
-            KeyValuePairT,
-            ReduceBySegmentOpT,
-            ScanTileStateT>
-        TilePrefixCallbackOpT;
-
-    // Shared memory type for this thread block
-    union _TempStorage
-    {
-        struct
-        {
-            typename BlockScanT::TempStorage                scan;           // Smem needed for tile scanning
-            typename TilePrefixCallbackOpT::TempStorage     prefix;         // Smem needed for cooperative prefix callback
-        };
-
-        // Smem needed for loading keys
-        typename BlockLoadPairs::TempStorage load_pairs;
-    };
-
-    // Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    //---------------------------------------------------------------------
-    // Per-thread fields
-    //---------------------------------------------------------------------
-
-    _TempStorage&                   temp_storage;       ///< Reference to temp_storage
-    WrappedPairsInputIteratorT      d_pairs_in;          ///< Input keys
-    AggregatesOutputIteratorT       d_aggregates_out;   ///< Output value aggregates
-    WrappedFixupInputIteratorT      d_fixup_in;         ///< Fixup input values
-    InequalityWrapper<EqualityOpT>  inequality_op;      ///< KeyT inequality operator
-    ReductionOpT                    reduction_op;       ///< Reduction operator
-    ReduceBySegmentOpT              scan_op;            ///< Reduce-by-segment scan operator
-
-
-    //---------------------------------------------------------------------
-    // Constructor
-    //---------------------------------------------------------------------
-
-    // Constructor
-    __device__ __forceinline__
-    AgentSegmentFixup(
-        TempStorage&                temp_storage,       ///< Reference to temp_storage
-        PairsInputIteratorT         d_pairs_in,          ///< Input keys
-        AggregatesOutputIteratorT   d_aggregates_out,   ///< Output value aggregates
-        EqualityOpT                 equality_op,        ///< KeyT equality operator
-        ReductionOpT                reduction_op)       ///< ValueT reduction operator
-    :
-        temp_storage(temp_storage.Alias()),
-        d_pairs_in(d_pairs_in),
-        d_aggregates_out(d_aggregates_out),
-        d_fixup_in(d_aggregates_out),
-        inequality_op(equality_op),
-        reduction_op(reduction_op),
-        scan_op(reduction_op)
-    {}
-
-
-    //---------------------------------------------------------------------
-    // Cooperatively scan a device-wide sequence of tiles with other CTAs
-    //---------------------------------------------------------------------
-
-
-    /**
-     * Process input tile.  Specialized for atomic-fixup
-     */
-    template <bool IS_LAST_TILE>
-    __device__ __forceinline__ void ConsumeTile(
-        OffsetT             num_remaining,      ///< Number of global input items remaining (including this tile)
-        int                 tile_idx,           ///< Tile index
-        OffsetT             tile_offset,        ///< Tile offset
-        ScanTileStateT&     tile_state,         ///< Global tile state descriptor
-        Int2Type<true>      use_atomic_fixup)   ///< Marker whether to use atomicAdd (instead of reduce-by-key)
-    {
-        KeyValuePairT   pairs[ITEMS_PER_THREAD];
-
-        // Load pairs
-        KeyValuePairT oob_pair;
-        oob_pair.key = -1;
-
-        if (IS_LAST_TILE)
-            BlockLoadPairs(temp_storage.load_pairs).Load(d_pairs_in + tile_offset, pairs, num_remaining, oob_pair);
-        else
-            BlockLoadPairs(temp_storage.load_pairs).Load(d_pairs_in + tile_offset, pairs);
-
-        // RLE 
-        #pragma unroll
-        for (int ITEM = 1; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            ValueT* d_scatter = d_aggregates_out + pairs[ITEM - 1].key;
-            if (pairs[ITEM].key != pairs[ITEM - 1].key)
-                atomicAdd(d_scatter, pairs[ITEM - 1].value);
-            else
-                pairs[ITEM].value = reduction_op(pairs[ITEM - 1].value, pairs[ITEM].value);
-        }
-
-        // Flush last item if valid
-        ValueT* d_scatter = d_aggregates_out + pairs[ITEMS_PER_THREAD - 1].key;
-        if ((!IS_LAST_TILE) || (pairs[ITEMS_PER_THREAD - 1].key >= 0))
-            atomicAdd(d_scatter, pairs[ITEMS_PER_THREAD - 1].value);
-    }
-
-
-    /**
-     * Process input tile.  Specialized for reduce-by-key fixup
-     */
-    template <bool IS_LAST_TILE>
-    __device__ __forceinline__ void ConsumeTile(
-        OffsetT             num_remaining,      ///< Number of global input items remaining (including this tile)
-        int                 tile_idx,           ///< Tile index
-        OffsetT             tile_offset,        ///< Tile offset
-        ScanTileStateT&     tile_state,         ///< Global tile state descriptor
-        Int2Type<false>     use_atomic_fixup)   ///< Marker whether to use atomicAdd (instead of reduce-by-key)
-    {
-        KeyValuePairT   pairs[ITEMS_PER_THREAD];
-        KeyValuePairT   scatter_pairs[ITEMS_PER_THREAD];
-
-        // Load pairs
-        KeyValuePairT oob_pair;
-        oob_pair.key = -1;
-
-        if (IS_LAST_TILE)
-            BlockLoadPairs(temp_storage.load_pairs).Load(d_pairs_in + tile_offset, pairs, num_remaining, oob_pair);
-        else
-            BlockLoadPairs(temp_storage.load_pairs).Load(d_pairs_in + tile_offset, pairs);
-
-        CTA_SYNC();
-
-        KeyValuePairT tile_aggregate;
-        if (tile_idx == 0)
-        {
-            // Exclusive scan of values and segment_flags
-            BlockScanT(temp_storage.scan).ExclusiveScan(pairs, scatter_pairs, scan_op, tile_aggregate);
-
-            // Update tile status if this is not the last tile
-            if (threadIdx.x == 0)
-            {
-                // Set first segment id to not trigger a flush (invalid from exclusive scan)
-                scatter_pairs[0].key = pairs[0].key;
-
-                if (!IS_LAST_TILE)
-                    tile_state.SetInclusive(0, tile_aggregate);
-
-            }
-        }
-        else
-        {
-            // Exclusive scan of values and segment_flags
-            TilePrefixCallbackOpT prefix_op(tile_state, temp_storage.prefix, scan_op, tile_idx);
-            BlockScanT(temp_storage.scan).ExclusiveScan(pairs, scatter_pairs, scan_op, prefix_op);
-            tile_aggregate = prefix_op.GetBlockAggregate();
-        }
-
-        // Scatter updated values
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            if (scatter_pairs[ITEM].key != pairs[ITEM].key)
-            {
-                // Update the value at the key location
-                ValueT value    = d_fixup_in[scatter_pairs[ITEM].key];
-                value           = reduction_op(value, scatter_pairs[ITEM].value);
-
-                d_aggregates_out[scatter_pairs[ITEM].key] = value;
-            }
-        }
-
-        // Finalize the last item
-        if (IS_LAST_TILE)
-        {
-            // Last thread will output final count and last item, if necessary
-            if (threadIdx.x == BLOCK_THREADS - 1)
-            {
-                // If the last tile is a whole tile, the inclusive prefix contains accumulated value reduction for the last segment
-                if (num_remaining == TILE_ITEMS)
-                {
-                    // Update the value at the key location
-                    OffsetT last_key = pairs[ITEMS_PER_THREAD - 1].key;
-                    d_aggregates_out[last_key] = reduction_op(tile_aggregate.value, d_fixup_in[last_key]);
-                }
-            }
-        }
-    }
-
-
-    /**
-     * Scan tiles of items as part of a dynamic chained scan
-     */
-    __device__ __forceinline__ void ConsumeRange(
-        int                 num_items,          ///< Total number of input items
-        int                 num_tiles,          ///< Total number of input tiles
-        ScanTileStateT&     tile_state)         ///< Global tile state descriptor
-    {
-        // Blocks are launched in increasing order, so just assign one tile per block
-        int     tile_idx        = (blockIdx.x * gridDim.y) + blockIdx.y;    // Current tile index
-        OffsetT tile_offset     = tile_idx * TILE_ITEMS;                    // Global offset for the current tile
-        OffsetT num_remaining   = num_items - tile_offset;                  // Remaining items (including this tile)
-
-        if (num_remaining > TILE_ITEMS)
-        {
-            // Not the last tile (full)
-            ConsumeTile<false>(num_remaining, tile_idx, tile_offset, tile_state, Int2Type<USE_ATOMIC_FIXUP>());
-        }
-        else if (num_remaining > 0)
-        {
-            // The last tile (possibly partially-full)
-            ConsumeTile<true>(num_remaining, tile_idx, tile_offset, tile_state, Int2Type<USE_ATOMIC_FIXUP>());
-        }
-    }
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/agent/agent_select_if.cuh b/pfsimulator/third_party/cub/agent/agent_select_if.cuh
deleted file mode 100644
index 52ca9fc28..000000000
--- a/pfsimulator/third_party/cub/agent/agent_select_if.cuh
+++ /dev/null
@@ -1,703 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::AgentSelectIf implements a stateful abstraction of CUDA thread blocks for participating in device-wide select.
- */
-
-#pragma once
-
-#include <iterator>
-
-#include "single_pass_scan_operators.cuh"
-#include "../block/block_load.cuh"
-#include "../block/block_store.cuh"
-#include "../block/block_scan.cuh"
-#include "../block/block_exchange.cuh"
-#include "../block/block_discontinuity.cuh"
-#include "../grid/grid_queue.cuh"
-#include "../iterator/cache_modified_input_iterator.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/******************************************************************************
- * Tuning policy types
- ******************************************************************************/
-
-/**
- * Parameterizable tuning policy type for AgentSelectIf
- */
-template <
-    int                         _BLOCK_THREADS,                 ///< Threads per thread block
-    int                         _ITEMS_PER_THREAD,              ///< Items per thread (per tile of input)
-    BlockLoadAlgorithm          _LOAD_ALGORITHM,                ///< The BlockLoad algorithm to use
-    CacheLoadModifier           _LOAD_MODIFIER,                 ///< Cache load modifier for reading input elements
-    BlockScanAlgorithm          _SCAN_ALGORITHM>                ///< The BlockScan algorithm to use
-struct AgentSelectIfPolicy
-{
-    enum
-    {
-        BLOCK_THREADS           = _BLOCK_THREADS,               ///< Threads per thread block
-        ITEMS_PER_THREAD        = _ITEMS_PER_THREAD,            ///< Items per thread (per tile of input)
-    };
-
-    static const BlockLoadAlgorithm     LOAD_ALGORITHM          = _LOAD_ALGORITHM;      ///< The BlockLoad algorithm to use
-    static const CacheLoadModifier      LOAD_MODIFIER           = _LOAD_MODIFIER;       ///< Cache load modifier for reading input elements
-    static const BlockScanAlgorithm     SCAN_ALGORITHM          = _SCAN_ALGORITHM;      ///< The BlockScan algorithm to use
-};
-
-
-
-
-/******************************************************************************
- * Thread block abstractions
- ******************************************************************************/
-
-
-/**
- * \brief AgentSelectIf implements a stateful abstraction of CUDA thread blocks for participating in device-wide selection
- *
- * Performs functor-based selection if SelectOpT functor type != NullType
- * Otherwise performs flag-based selection if FlagsInputIterator's value type != NullType
- * Otherwise performs discontinuity selection (keep unique)
- */
-template <
-    typename    AgentSelectIfPolicyT,           ///< Parameterized AgentSelectIfPolicy tuning policy type
-    typename    InputIteratorT,                 ///< Random-access input iterator type for selection items
-    typename    FlagsInputIteratorT,            ///< Random-access input iterator type for selections (NullType* if a selection functor or discontinuity flagging is to be used for selection)
-    typename    SelectedOutputIteratorT,        ///< Random-access input iterator type for selection_flags items
-    typename    SelectOpT,                      ///< Selection operator type (NullType if selections or discontinuity flagging is to be used for selection)
-    typename    EqualityOpT,                    ///< Equality operator type (NullType if selection functor or selections is to be used for selection)
-    typename    OffsetT,                        ///< Signed integer type for global offsets
-    bool        KEEP_REJECTS>                   ///< Whether or not we push rejected items to the back of the output
-struct AgentSelectIf
-{
-    //---------------------------------------------------------------------
-    // Types and constants
-    //---------------------------------------------------------------------
-
-    // The input value type
-    typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
-
-    // The output value type
-    typedef typename If<(Equals<typename std::iterator_traits<SelectedOutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
-        typename std::iterator_traits<InputIteratorT>::value_type,                                                  // ... then the input iterator's value type,
-        typename std::iterator_traits<SelectedOutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
-
-    // The flag value type
-    typedef typename std::iterator_traits<FlagsInputIteratorT>::value_type FlagT;
-
-    // Tile status descriptor interface type
-    typedef ScanTileState<OffsetT> ScanTileStateT;
-
-    // Constants
-    enum
-    {
-        USE_SELECT_OP,
-        USE_SELECT_FLAGS,
-        USE_DISCONTINUITY,
-
-        BLOCK_THREADS           = AgentSelectIfPolicyT::BLOCK_THREADS,
-        ITEMS_PER_THREAD        = AgentSelectIfPolicyT::ITEMS_PER_THREAD,
-        TILE_ITEMS              = BLOCK_THREADS * ITEMS_PER_THREAD,
-        TWO_PHASE_SCATTER       = (ITEMS_PER_THREAD > 1),
-
-        SELECT_METHOD           = (!Equals<SelectOpT, NullType>::VALUE) ?
-                                    USE_SELECT_OP :
-                                    (!Equals<FlagT, NullType>::VALUE) ?
-                                        USE_SELECT_FLAGS :
-                                        USE_DISCONTINUITY
-    };
-
-    // Cache-modified Input iterator wrapper type (for applying cache modifier) for items
-    typedef typename If<IsPointer<InputIteratorT>::VALUE,
-            CacheModifiedInputIterator<AgentSelectIfPolicyT::LOAD_MODIFIER, InputT, OffsetT>,        // Wrap the native input pointer with CacheModifiedValuesInputIterator
-            InputIteratorT>::Type                                                               // Directly use the supplied input iterator type
-        WrappedInputIteratorT;
-
-    // Cache-modified Input iterator wrapper type (for applying cache modifier) for values
-    typedef typename If<IsPointer<FlagsInputIteratorT>::VALUE,
-            CacheModifiedInputIterator<AgentSelectIfPolicyT::LOAD_MODIFIER, FlagT, OffsetT>,    // Wrap the native input pointer with CacheModifiedValuesInputIterator
-            FlagsInputIteratorT>::Type                                                          // Directly use the supplied input iterator type
-        WrappedFlagsInputIteratorT;
-
-    // Parameterized BlockLoad type for input data
-    typedef BlockLoad<
-            OutputT,
-            BLOCK_THREADS,
-            ITEMS_PER_THREAD,
-            AgentSelectIfPolicyT::LOAD_ALGORITHM>
-        BlockLoadT;
-
-    // Parameterized BlockLoad type for flags
-    typedef BlockLoad<
-            FlagT,
-            BLOCK_THREADS,
-            ITEMS_PER_THREAD,
-            AgentSelectIfPolicyT::LOAD_ALGORITHM>
-        BlockLoadFlags;
-
-    // Parameterized BlockDiscontinuity type for items
-    typedef BlockDiscontinuity<
-            OutputT,
-            BLOCK_THREADS>
-        BlockDiscontinuityT;
-
-    // Parameterized BlockScan type
-    typedef BlockScan<
-            OffsetT,
-            BLOCK_THREADS,
-            AgentSelectIfPolicyT::SCAN_ALGORITHM>
-        BlockScanT;
-
-    // Callback type for obtaining tile prefix during block scan
-    typedef TilePrefixCallbackOp<
-            OffsetT,
-            cub::Sum,
-            ScanTileStateT>
-        TilePrefixCallbackOpT;
-
-    // Item exchange type
-    typedef OutputT ItemExchangeT[TILE_ITEMS];
-
-    // Shared memory type for this thread block
-    union _TempStorage
-    {
-        struct
-        {
-            typename BlockScanT::TempStorage                scan;           // Smem needed for tile scanning
-            typename TilePrefixCallbackOpT::TempStorage     prefix;         // Smem needed for cooperative prefix callback
-            typename BlockDiscontinuityT::TempStorage       discontinuity;  // Smem needed for discontinuity detection
-        };
-
-        // Smem needed for loading items
-        typename BlockLoadT::TempStorage load_items;
-
-        // Smem needed for loading values
-        typename BlockLoadFlags::TempStorage load_flags;
-
-        // Smem needed for compacting items (allows non POD items in this union)
-        Uninitialized<ItemExchangeT> raw_exchange;
-    };
-
-    // Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    //---------------------------------------------------------------------
-    // Per-thread fields
-    //---------------------------------------------------------------------
-
-    _TempStorage&                   temp_storage;       ///< Reference to temp_storage
-    WrappedInputIteratorT           d_in;               ///< Input items
-    SelectedOutputIteratorT         d_selected_out;     ///< Unique output items
-    WrappedFlagsInputIteratorT      d_flags_in;         ///< Input selection flags (if applicable)
-    InequalityWrapper<EqualityOpT>  inequality_op;      ///< T inequality operator
-    SelectOpT                       select_op;          ///< Selection operator
-    OffsetT                         num_items;          ///< Total number of input items
-
-
-    //---------------------------------------------------------------------
-    // Constructor
-    //---------------------------------------------------------------------
-
-    // Constructor
-    __device__ __forceinline__
-    AgentSelectIf(
-        TempStorage                 &temp_storage,      ///< Reference to temp_storage
-        InputIteratorT              d_in,               ///< Input data
-        FlagsInputIteratorT         d_flags_in,         ///< Input selection flags (if applicable)
-        SelectedOutputIteratorT     d_selected_out,     ///< Output data
-        SelectOpT                   select_op,          ///< Selection operator
-        EqualityOpT                 equality_op,        ///< Equality operator
-        OffsetT                     num_items)          ///< Total number of input items
-    :
-        temp_storage(temp_storage.Alias()),
-        d_in(d_in),
-        d_flags_in(d_flags_in),
-        d_selected_out(d_selected_out),
-        select_op(select_op),
-        inequality_op(equality_op),
-        num_items(num_items)
-    {}
-
-
-    //---------------------------------------------------------------------
-    // Utility methods for initializing the selections
-    //---------------------------------------------------------------------
-
-    /**
-     * Initialize selections (specialized for selection operator)
-     */
-    template <bool IS_FIRST_TILE, bool IS_LAST_TILE>
-    __device__ __forceinline__ void InitializeSelections(
-        OffsetT                     /*tile_offset*/,
-        OffsetT                     num_tile_items,
-        OutputT                     (&items)[ITEMS_PER_THREAD],
-        OffsetT                     (&selection_flags)[ITEMS_PER_THREAD],
-        Int2Type<USE_SELECT_OP>     /*select_method*/)
-    {
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            // Out-of-bounds items are selection_flags
-            selection_flags[ITEM] = 1;
-
-            if (!IS_LAST_TILE || (OffsetT(threadIdx.x * ITEMS_PER_THREAD) + ITEM < num_tile_items))
-                selection_flags[ITEM] = select_op(items[ITEM]);
-        }
-    }
-
-
-    /**
-     * Initialize selections (specialized for valid flags)
-     */
-    template <bool IS_FIRST_TILE, bool IS_LAST_TILE>
-    __device__ __forceinline__ void InitializeSelections(
-        OffsetT                     tile_offset,
-        OffsetT                     num_tile_items,
-        OutputT                     (&/*items*/)[ITEMS_PER_THREAD],
-        OffsetT                     (&selection_flags)[ITEMS_PER_THREAD],
-        Int2Type<USE_SELECT_FLAGS>  /*select_method*/)
-    {
-        CTA_SYNC();
-
-        FlagT flags[ITEMS_PER_THREAD];
-
-        if (IS_LAST_TILE)
-        {
-            // Out-of-bounds items are selection_flags
-            BlockLoadFlags(temp_storage.load_flags).Load(d_flags_in + tile_offset, flags, num_tile_items, 1);
-        }
-        else
-        {
-            BlockLoadFlags(temp_storage.load_flags).Load(d_flags_in + tile_offset, flags);
-        }
-
-        // Convert flag type to selection_flags type
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            selection_flags[ITEM] = flags[ITEM];
-        }
-    }
-
-
-    /**
-     * Initialize selections (specialized for discontinuity detection)
-     */
-    template <bool IS_FIRST_TILE, bool IS_LAST_TILE>
-    __device__ __forceinline__ void InitializeSelections(
-        OffsetT                     tile_offset,
-        OffsetT                     num_tile_items,
-        OutputT                     (&items)[ITEMS_PER_THREAD],
-        OffsetT                     (&selection_flags)[ITEMS_PER_THREAD],
-        Int2Type<USE_DISCONTINUITY> /*select_method*/)
-    {
-        if (IS_FIRST_TILE)
-        {
-            CTA_SYNC();
-
-            // Set head selection_flags.  First tile sets the first flag for the first item
-            BlockDiscontinuityT(temp_storage.discontinuity).FlagHeads(selection_flags, items, inequality_op);
-        }
-        else
-        {
-            OutputT tile_predecessor;
-            if (threadIdx.x == 0)
-                tile_predecessor = d_in[tile_offset - 1];
-
-            CTA_SYNC();
-
-            BlockDiscontinuityT(temp_storage.discontinuity).FlagHeads(selection_flags, items, inequality_op, tile_predecessor);
-        }
-
-        // Set selection flags for out-of-bounds items
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            // Set selection_flags for out-of-bounds items
-            if ((IS_LAST_TILE) && (OffsetT(threadIdx.x * ITEMS_PER_THREAD) + ITEM >= num_tile_items))
-                selection_flags[ITEM] = 1;
-        }
-    }
-
-
-    //---------------------------------------------------------------------
-    // Scatter utility methods
-    //---------------------------------------------------------------------
-
-    /**
-     * Scatter flagged items to output offsets (specialized for direct scattering)
-     */
-    template <bool IS_LAST_TILE, bool IS_FIRST_TILE>
-    __device__ __forceinline__ void ScatterDirect(
-        OutputT (&items)[ITEMS_PER_THREAD],
-        OffsetT (&selection_flags)[ITEMS_PER_THREAD],
-        OffsetT (&selection_indices)[ITEMS_PER_THREAD],
-        OffsetT num_selections)
-    {
-        // Scatter flagged items
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            if (selection_flags[ITEM])
-            {
-                if ((!IS_LAST_TILE) || selection_indices[ITEM] < num_selections)
-                {
-                    d_selected_out[selection_indices[ITEM]] = items[ITEM];
-                }
-            }
-        }
-    }
-
-
-    /**
-     * Scatter flagged items to output offsets (specialized for two-phase scattering)
-     */
-    template <bool IS_LAST_TILE, bool IS_FIRST_TILE>
-    __device__ __forceinline__ void ScatterTwoPhase(
-        OutputT         (&items)[ITEMS_PER_THREAD],
-        OffsetT         (&selection_flags)[ITEMS_PER_THREAD],
-        OffsetT         (&selection_indices)[ITEMS_PER_THREAD],
-        int             /*num_tile_items*/,                         ///< Number of valid items in this tile
-        int             num_tile_selections,                        ///< Number of selections in this tile
-        OffsetT         num_selections_prefix,                      ///< Total number of selections prior to this tile
-        OffsetT         /*num_rejected_prefix*/,                    ///< Total number of rejections prior to this tile
-        Int2Type<false> /*is_keep_rejects*/)                        ///< Marker type indicating whether to keep rejected items in the second partition
-    {
-        CTA_SYNC();
-
-        // Compact and scatter items
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            int local_scatter_offset = selection_indices[ITEM] - num_selections_prefix;
-            if (selection_flags[ITEM])
-            {
-                temp_storage.raw_exchange.Alias()[local_scatter_offset] = items[ITEM];
-            }
-        }
-
-        CTA_SYNC();
-
-        for (int item = threadIdx.x; item < num_tile_selections; item += BLOCK_THREADS)
-        {
-            d_selected_out[num_selections_prefix + item] = temp_storage.raw_exchange.Alias()[item];
-        }
-    }
-
-
-    /**
-     * Scatter flagged items to output offsets (specialized for two-phase scattering)
-     */
-    template <bool IS_LAST_TILE, bool IS_FIRST_TILE>
-    __device__ __forceinline__ void ScatterTwoPhase(
-        OutputT         (&items)[ITEMS_PER_THREAD],
-        OffsetT         (&selection_flags)[ITEMS_PER_THREAD],
-        OffsetT         (&selection_indices)[ITEMS_PER_THREAD],
-        int             num_tile_items,                             ///< Number of valid items in this tile
-        int             num_tile_selections,                        ///< Number of selections in this tile
-        OffsetT         num_selections_prefix,                      ///< Total number of selections prior to this tile
-        OffsetT         num_rejected_prefix,                        ///< Total number of rejections prior to this tile
-        Int2Type<true>  /*is_keep_rejects*/)                        ///< Marker type indicating whether to keep rejected items in the second partition
-    {
-        CTA_SYNC();
-
-        int tile_num_rejections = num_tile_items - num_tile_selections;
-
-        // Scatter items to shared memory (rejections first)
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            int item_idx                = (threadIdx.x * ITEMS_PER_THREAD) + ITEM;
-            int local_selection_idx     = selection_indices[ITEM] - num_selections_prefix;
-            int local_rejection_idx     = item_idx - local_selection_idx;
-            int local_scatter_offset    = (selection_flags[ITEM]) ?
-                                            tile_num_rejections + local_selection_idx :
-                                            local_rejection_idx;
-
-            temp_storage.raw_exchange.Alias()[local_scatter_offset] = items[ITEM];
-        }
-
-        CTA_SYNC();
-
-        // Gather items from shared memory and scatter to global
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            int item_idx            = (ITEM * BLOCK_THREADS) + threadIdx.x;
-            int rejection_idx       = item_idx;
-            int selection_idx       = item_idx - tile_num_rejections;
-            OffsetT scatter_offset  = (item_idx < tile_num_rejections) ?
-                                        num_items - num_rejected_prefix - rejection_idx - 1 :
-                                        num_selections_prefix + selection_idx;
-
-            OutputT item = temp_storage.raw_exchange.Alias()[item_idx];
-
-            if (!IS_LAST_TILE || (item_idx < num_tile_items))
-            {
-                d_selected_out[scatter_offset] = item;
-            }
-        }
-    }
-
-
-    /**
-     * Scatter flagged items
-     */
-    template <bool IS_LAST_TILE, bool IS_FIRST_TILE>
-    __device__ __forceinline__ void Scatter(
-        OutputT         (&items)[ITEMS_PER_THREAD],
-        OffsetT         (&selection_flags)[ITEMS_PER_THREAD],
-        OffsetT         (&selection_indices)[ITEMS_PER_THREAD],
-        int             num_tile_items,                             ///< Number of valid items in this tile
-        int             num_tile_selections,                        ///< Number of selections in this tile
-        OffsetT         num_selections_prefix,                      ///< Total number of selections prior to this tile
-        OffsetT         num_rejected_prefix,                        ///< Total number of rejections prior to this tile
-        OffsetT         num_selections)                             ///< Total number of selections including this tile
-    {
-        // Do a two-phase scatter if (a) keeping both partitions or (b) two-phase is enabled and the average number of selection_flags items per thread is greater than one
-        if (KEEP_REJECTS || (TWO_PHASE_SCATTER && (num_tile_selections > BLOCK_THREADS)))
-        {
-            ScatterTwoPhase<IS_LAST_TILE, IS_FIRST_TILE>(
-                items,
-                selection_flags,
-                selection_indices,
-                num_tile_items,
-                num_tile_selections,
-                num_selections_prefix,
-                num_rejected_prefix,
-                Int2Type<KEEP_REJECTS>());
-        }
-        else
-        {
-            ScatterDirect<IS_LAST_TILE, IS_FIRST_TILE>(
-                items,
-                selection_flags,
-                selection_indices,
-                num_selections);
-        }
-    }
-
-    //---------------------------------------------------------------------
-    // Cooperatively scan a device-wide sequence of tiles with other CTAs
-    //---------------------------------------------------------------------
-
-
-    /**
-     * Process first tile of input (dynamic chained scan).  Returns the running count of selections (including this tile)
-     */
-    template <bool IS_LAST_TILE>
-    __device__ __forceinline__ OffsetT ConsumeFirstTile(
-        int                 num_tile_items,      ///< Number of input items comprising this tile
-        OffsetT             tile_offset,        ///< Tile offset
-        ScanTileStateT&     tile_state)         ///< Global tile state descriptor
-    {
-        OutputT     items[ITEMS_PER_THREAD];
-        OffsetT     selection_flags[ITEMS_PER_THREAD];
-        OffsetT     selection_indices[ITEMS_PER_THREAD];
-
-        // Load items
-        if (IS_LAST_TILE)
-            BlockLoadT(temp_storage.load_items).Load(d_in + tile_offset, items, num_tile_items);
-        else
-            BlockLoadT(temp_storage.load_items).Load(d_in + tile_offset, items);
-
-        // Initialize selection_flags
-        InitializeSelections<true, IS_LAST_TILE>(
-            tile_offset,
-            num_tile_items,
-            items,
-            selection_flags,
-            Int2Type<SELECT_METHOD>());
-
-        CTA_SYNC();
-
-        // Exclusive scan of selection_flags
-        OffsetT num_tile_selections;
-        BlockScanT(temp_storage.scan).ExclusiveSum(selection_flags, selection_indices, num_tile_selections);
-
-        if (threadIdx.x == 0)
-        {
-            // Update tile status if this is not the last tile
-            if (!IS_LAST_TILE)
-                tile_state.SetInclusive(0, num_tile_selections);
-        }
-
-        // Discount any out-of-bounds selections
-        if (IS_LAST_TILE)
-            num_tile_selections -= (TILE_ITEMS - num_tile_items);
-
-        // Scatter flagged items
-        Scatter<IS_LAST_TILE, true>(
-            items,
-            selection_flags,
-            selection_indices,
-            num_tile_items,
-            num_tile_selections,
-            0,
-            0,
-            num_tile_selections);
-
-        return num_tile_selections;
-    }
-
-
-    /**
-     * Process subsequent tile of input (dynamic chained scan).  Returns the running count of selections (including this tile)
-     */
-    template <bool IS_LAST_TILE>
-    __device__ __forceinline__ OffsetT ConsumeSubsequentTile(
-        int                 num_tile_items,      ///< Number of input items comprising this tile
-        int                 tile_idx,           ///< Tile index
-        OffsetT             tile_offset,        ///< Tile offset
-        ScanTileStateT&     tile_state)         ///< Global tile state descriptor
-    {
-        OutputT     items[ITEMS_PER_THREAD];
-        OffsetT     selection_flags[ITEMS_PER_THREAD];
-        OffsetT     selection_indices[ITEMS_PER_THREAD];
-
-        // Load items
-        if (IS_LAST_TILE)
-            BlockLoadT(temp_storage.load_items).Load(d_in + tile_offset, items, num_tile_items);
-        else
-            BlockLoadT(temp_storage.load_items).Load(d_in + tile_offset, items);
-
-        // Initialize selection_flags
-        InitializeSelections<false, IS_LAST_TILE>(
-            tile_offset,
-            num_tile_items,
-            items,
-            selection_flags,
-            Int2Type<SELECT_METHOD>());
-
-        CTA_SYNC();
-
-        // Exclusive scan of values and selection_flags
-        TilePrefixCallbackOpT prefix_op(tile_state, temp_storage.prefix, cub::Sum(), tile_idx);
-        BlockScanT(temp_storage.scan).ExclusiveSum(selection_flags, selection_indices, prefix_op);
-
-        OffsetT num_tile_selections     = prefix_op.GetBlockAggregate();
-        OffsetT num_selections          = prefix_op.GetInclusivePrefix();
-        OffsetT num_selections_prefix   = prefix_op.GetExclusivePrefix();
-        OffsetT num_rejected_prefix     = (tile_idx * TILE_ITEMS) - num_selections_prefix;
-
-        // Discount any out-of-bounds selections
-        if (IS_LAST_TILE)
-        {
-            int num_discount    = TILE_ITEMS - num_tile_items;
-            num_selections      -= num_discount;
-            num_tile_selections -= num_discount;
-        }
-
-        // Scatter flagged items
-        Scatter<IS_LAST_TILE, false>(
-            items,
-            selection_flags,
-            selection_indices,
-            num_tile_items,
-            num_tile_selections,
-            num_selections_prefix,
-            num_rejected_prefix,
-            num_selections);
-
-        return num_selections;
-    }
-
-
-    /**
-     * Process a tile of input
-     */
-    template <bool IS_LAST_TILE>
-    __device__ __forceinline__ OffsetT ConsumeTile(
-        int                 num_tile_items,         ///< Number of input items comprising this tile
-        int                 tile_idx,           ///< Tile index
-        OffsetT             tile_offset,        ///< Tile offset
-        ScanTileStateT&     tile_state)         ///< Global tile state descriptor
-    {
-        OffsetT num_selections;
-        if (tile_idx == 0)
-        {
-            num_selections = ConsumeFirstTile<IS_LAST_TILE>(num_tile_items, tile_offset, tile_state);
-        }
-        else
-        {
-            num_selections = ConsumeSubsequentTile<IS_LAST_TILE>(num_tile_items, tile_idx, tile_offset, tile_state);
-        }
-
-        return num_selections;
-    }
-
-
-    /**
-     * Scan tiles of items as part of a dynamic chained scan
-     */
-    template <typename NumSelectedIteratorT>        ///< Output iterator type for recording number of items selection_flags
-    __device__ __forceinline__ void ConsumeRange(
-        int                     num_tiles,          ///< Total number of input tiles
-        ScanTileStateT&         tile_state,         ///< Global tile state descriptor
-        NumSelectedIteratorT    d_num_selected_out) ///< Output total number selection_flags
-    {
-        // Blocks are launched in increasing order, so just assign one tile per block
-        int     tile_idx        = (blockIdx.x * gridDim.y) + blockIdx.y;    // Current tile index
-        OffsetT tile_offset     = tile_idx * TILE_ITEMS;                    // Global offset for the current tile
-
-        if (tile_idx < num_tiles - 1)
-        {
-            // Not the last tile (full)
-            ConsumeTile<false>(TILE_ITEMS, tile_idx, tile_offset, tile_state);
-        }
-        else
-        {
-            // The last tile (possibly partially-full)
-            OffsetT num_remaining   = num_items - tile_offset;
-            OffsetT num_selections  = ConsumeTile<true>(num_remaining, tile_idx, tile_offset, tile_state);
-
-            if (threadIdx.x == 0)
-            {
-                // Output the total number of items selection_flags
-                *d_num_selected_out = num_selections;
-            }
-        }
-    }
-
-};
-
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/agent/agent_spmv_orig.cuh b/pfsimulator/third_party/cub/agent/agent_spmv_orig.cuh
deleted file mode 100644
index 54e2a1394..000000000
--- a/pfsimulator/third_party/cub/agent/agent_spmv_orig.cuh
+++ /dev/null
@@ -1,670 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::AgentSpmv implements a stateful abstraction of CUDA thread blocks for participating in device-wide SpMV.
- */
-
-#pragma once
-
-#include <iterator>
-
-#include "../util_type.cuh"
-#include "../block/block_reduce.cuh"
-#include "../block/block_scan.cuh"
-#include "../block/block_exchange.cuh"
-#include "../thread/thread_search.cuh"
-#include "../thread/thread_operators.cuh"
-#include "../iterator/cache_modified_input_iterator.cuh"
-#include "../iterator/counting_input_iterator.cuh"
-#include "../iterator/tex_ref_input_iterator.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/******************************************************************************
- * Tuning policy
- ******************************************************************************/
-
-/**
- * Parameterizable tuning policy type for AgentSpmv
- */
-template <
-    int                             _BLOCK_THREADS,                         ///< Threads per thread block
-    int                             _ITEMS_PER_THREAD,                      ///< Items per thread (per tile of input)
-    CacheLoadModifier               _ROW_OFFSETS_SEARCH_LOAD_MODIFIER,      ///< Cache load modifier for reading CSR row-offsets during search
-    CacheLoadModifier               _ROW_OFFSETS_LOAD_MODIFIER,             ///< Cache load modifier for reading CSR row-offsets
-    CacheLoadModifier               _COLUMN_INDICES_LOAD_MODIFIER,          ///< Cache load modifier for reading CSR column-indices
-    CacheLoadModifier               _VALUES_LOAD_MODIFIER,                  ///< Cache load modifier for reading CSR values
-    CacheLoadModifier               _VECTOR_VALUES_LOAD_MODIFIER,           ///< Cache load modifier for reading vector values
-    bool                            _DIRECT_LOAD_NONZEROS,                  ///< Whether to load nonzeros directly from global during sequential merging (vs. pre-staged through shared memory)
-    BlockScanAlgorithm              _SCAN_ALGORITHM>                        ///< The BlockScan algorithm to use
-struct AgentSpmvPolicy
-{
-    enum
-    {
-        BLOCK_THREADS                                                   = _BLOCK_THREADS,                       ///< Threads per thread block
-        ITEMS_PER_THREAD                                                = _ITEMS_PER_THREAD,                    ///< Items per thread (per tile of input)
-        DIRECT_LOAD_NONZEROS                                            = _DIRECT_LOAD_NONZEROS,                ///< Whether to load nonzeros directly from global during sequential merging (pre-staged through shared memory)
-    };
-
-    static const CacheLoadModifier  ROW_OFFSETS_SEARCH_LOAD_MODIFIER    = _ROW_OFFSETS_SEARCH_LOAD_MODIFIER;    ///< Cache load modifier for reading CSR row-offsets
-    static const CacheLoadModifier  ROW_OFFSETS_LOAD_MODIFIER           = _ROW_OFFSETS_LOAD_MODIFIER;           ///< Cache load modifier for reading CSR row-offsets
-    static const CacheLoadModifier  COLUMN_INDICES_LOAD_MODIFIER        = _COLUMN_INDICES_LOAD_MODIFIER;        ///< Cache load modifier for reading CSR column-indices
-    static const CacheLoadModifier  VALUES_LOAD_MODIFIER                = _VALUES_LOAD_MODIFIER;                ///< Cache load modifier for reading CSR values
-    static const CacheLoadModifier  VECTOR_VALUES_LOAD_MODIFIER         = _VECTOR_VALUES_LOAD_MODIFIER;         ///< Cache load modifier for reading vector values
-    static const BlockScanAlgorithm SCAN_ALGORITHM                      = _SCAN_ALGORITHM;                      ///< The BlockScan algorithm to use
-
-};
-
-
-/******************************************************************************
- * Thread block abstractions
- ******************************************************************************/
-
-template <
-    typename        ValueT,              ///< Matrix and vector value type
-    typename        OffsetT>             ///< Signed integer type for sequence offsets
-struct SpmvParams
-{
-    ValueT*         d_values;            ///< Pointer to the array of \p num_nonzeros values of the corresponding nonzero elements of matrix <b>A</b>.
-    OffsetT*        d_row_end_offsets;   ///< Pointer to the array of \p m offsets demarcating the end of every row in \p d_column_indices and \p d_values
-    OffsetT*        d_column_indices;    ///< Pointer to the array of \p num_nonzeros column-indices of the corresponding nonzero elements of matrix <b>A</b>.  (Indices are zero-valued.)
-    ValueT*         d_vector_x;          ///< Pointer to the array of \p num_cols values corresponding to the dense input vector <em>x</em>
-    ValueT*         d_vector_y;          ///< Pointer to the array of \p num_rows values corresponding to the dense output vector <em>y</em>
-    int             num_rows;            ///< Number of rows of matrix <b>A</b>.
-    int             num_cols;            ///< Number of columns of matrix <b>A</b>.
-    int             num_nonzeros;        ///< Number of nonzero elements of matrix <b>A</b>.
-    ValueT          alpha;               ///< Alpha multiplicand
-    ValueT          beta;                ///< Beta addend-multiplicand
-
-    TexRefInputIterator<ValueT, 66778899, OffsetT>  t_vector_x;
-};
-
-
-/**
- * \brief AgentSpmv implements a stateful abstraction of CUDA thread blocks for participating in device-wide SpMV.
- */
-template <
-    typename    AgentSpmvPolicyT,           ///< Parameterized AgentSpmvPolicy tuning policy type
-    typename    ValueT,                     ///< Matrix and vector value type
-    typename    OffsetT,                    ///< Signed integer type for sequence offsets
-    bool        HAS_ALPHA,                  ///< Whether the input parameter \p alpha is 1
-    bool        HAS_BETA,                   ///< Whether the input parameter \p beta is 0
-    int         PTX_ARCH = CUB_PTX_ARCH>    ///< PTX compute capability
-struct AgentSpmv
-{
-    //---------------------------------------------------------------------
-    // Types and constants
-    //---------------------------------------------------------------------
-
-    /// Constants
-    enum
-    {
-        BLOCK_THREADS           = AgentSpmvPolicyT::BLOCK_THREADS,
-        ITEMS_PER_THREAD        = AgentSpmvPolicyT::ITEMS_PER_THREAD,
-        TILE_ITEMS              = BLOCK_THREADS * ITEMS_PER_THREAD,
-    };
-
-    /// 2D merge path coordinate type
-    typedef typename CubVector<OffsetT, 2>::Type CoordinateT;
-
-    /// Input iterator wrapper types (for applying cache modifiers)
-
-    typedef CacheModifiedInputIterator<
-            AgentSpmvPolicyT::ROW_OFFSETS_SEARCH_LOAD_MODIFIER,
-            OffsetT,
-            OffsetT>
-        RowOffsetsSearchIteratorT;
-
-    typedef CacheModifiedInputIterator<
-            AgentSpmvPolicyT::ROW_OFFSETS_LOAD_MODIFIER,
-            OffsetT,
-            OffsetT>
-        RowOffsetsIteratorT;
-
-    typedef CacheModifiedInputIterator<
-            AgentSpmvPolicyT::COLUMN_INDICES_LOAD_MODIFIER,
-            OffsetT,
-            OffsetT>
-        ColumnIndicesIteratorT;
-
-    typedef CacheModifiedInputIterator<
-            AgentSpmvPolicyT::VALUES_LOAD_MODIFIER,
-            ValueT,
-            OffsetT>
-        ValueIteratorT;
-
-    typedef CacheModifiedInputIterator<
-            AgentSpmvPolicyT::VECTOR_VALUES_LOAD_MODIFIER,
-            ValueT,
-            OffsetT>
-        VectorValueIteratorT;
-
-    // Tuple type for scanning (pairs accumulated segment-value with segment-index)
-    typedef KeyValuePair<OffsetT, ValueT> KeyValuePairT;
-
-    // Reduce-value-by-segment scan operator
-    typedef ReduceByKeyOp<cub::Sum> ReduceBySegmentOpT;
-
-    // BlockReduce specialization
-    typedef BlockReduce<
-            ValueT,
-            BLOCK_THREADS,
-            BLOCK_REDUCE_WARP_REDUCTIONS>
-        BlockReduceT;
-
-    // BlockScan specialization
-    typedef BlockScan<
-            KeyValuePairT,
-            BLOCK_THREADS,
-            AgentSpmvPolicyT::SCAN_ALGORITHM>
-        BlockScanT;
-
-    // BlockScan specialization
-    typedef BlockScan<
-            ValueT,
-            BLOCK_THREADS,
-            AgentSpmvPolicyT::SCAN_ALGORITHM>
-        BlockPrefixSumT;
-
-    // BlockExchange specialization
-    typedef BlockExchange<
-            ValueT,
-            BLOCK_THREADS,
-            ITEMS_PER_THREAD>
-        BlockExchangeT;
-
-    /// Merge item type (either a non-zero value or a row-end offset)
-    union MergeItem
-    {
-        // Value type to pair with index type OffsetT (NullType if loading values directly during merge)
-        typedef typename If<AgentSpmvPolicyT::DIRECT_LOAD_NONZEROS, NullType, ValueT>::Type MergeValueT;
-
-        OffsetT     row_end_offset;
-        MergeValueT nonzero;
-    };
-
-    /// Shared memory type required by this thread block
-    struct _TempStorage
-    {
-        CoordinateT tile_coords[2];
-
-        union Aliasable
-        {
-            // Smem needed for tile of merge items
-            MergeItem merge_items[ITEMS_PER_THREAD + TILE_ITEMS + 1];
-
-            // Smem needed for block exchange
-            typename BlockExchangeT::TempStorage exchange;
-
-            // Smem needed for block-wide reduction
-            typename BlockReduceT::TempStorage reduce;
-
-            // Smem needed for tile scanning
-            typename BlockScanT::TempStorage scan;
-
-            // Smem needed for tile prefix sum
-            typename BlockPrefixSumT::TempStorage prefix_sum;
-
-        } aliasable;
-    };
-
-    /// Temporary storage type (unionable)
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    //---------------------------------------------------------------------
-    // Per-thread fields
-    //---------------------------------------------------------------------
-
-
-    _TempStorage&                   temp_storage;         /// Reference to temp_storage
-
-    SpmvParams<ValueT, OffsetT>&    spmv_params;
-
-    ValueIteratorT                  wd_values;            ///< Wrapped pointer to the array of \p num_nonzeros values of the corresponding nonzero elements of matrix <b>A</b>.
-    RowOffsetsIteratorT             wd_row_end_offsets;   ///< Wrapped Pointer to the array of \p m offsets demarcating the end of every row in \p d_column_indices and \p d_values
-    ColumnIndicesIteratorT          wd_column_indices;    ///< Wrapped Pointer to the array of \p num_nonzeros column-indices of the corresponding nonzero elements of matrix <b>A</b>.  (Indices are zero-valued.)
-    VectorValueIteratorT            wd_vector_x;          ///< Wrapped Pointer to the array of \p num_cols values corresponding to the dense input vector <em>x</em>
-    VectorValueIteratorT            wd_vector_y;          ///< Wrapped Pointer to the array of \p num_cols values corresponding to the dense input vector <em>x</em>
-
-
-    //---------------------------------------------------------------------
-    // Interface
-    //---------------------------------------------------------------------
-
-    /**
-     * Constructor
-     */
-    __device__ __forceinline__ AgentSpmv(
-        TempStorage&                    temp_storage,           ///< Reference to temp_storage
-        SpmvParams<ValueT, OffsetT>&    spmv_params)            ///< SpMV input parameter bundle
-    :
-        temp_storage(temp_storage.Alias()),
-        spmv_params(spmv_params),
-        wd_values(spmv_params.d_values),
-        wd_row_end_offsets(spmv_params.d_row_end_offsets),
-        wd_column_indices(spmv_params.d_column_indices),
-        wd_vector_x(spmv_params.d_vector_x),
-        wd_vector_y(spmv_params.d_vector_y)
-    {}
-
-
-
-
-    /**
-     * Consume a merge tile, specialized for direct-load of nonzeros
-     */
-    __device__ __forceinline__ KeyValuePairT ConsumeTile(
-        int             tile_idx,
-        CoordinateT     tile_start_coord,
-        CoordinateT     tile_end_coord,
-        Int2Type<true>  is_direct_load)     ///< Marker type indicating whether to load nonzeros directly during path-discovery or beforehand in batch
-    {
-        int         tile_num_rows           = tile_end_coord.x - tile_start_coord.x;
-        int         tile_num_nonzeros       = tile_end_coord.y - tile_start_coord.y;
-        OffsetT*    s_tile_row_end_offsets  = &temp_storage.aliasable.merge_items[0].row_end_offset;
-
-        // Gather the row end-offsets for the merge tile into shared memory
-        for (int item = threadIdx.x; item <= tile_num_rows; item += BLOCK_THREADS)
-        {
-            s_tile_row_end_offsets[item] = wd_row_end_offsets[tile_start_coord.x + item];
-        }
-
-        CTA_SYNC();
-
-        // Search for the thread's starting coordinate within the merge tile
-        CountingInputIterator<OffsetT>  tile_nonzero_indices(tile_start_coord.y);
-        CoordinateT                     thread_start_coord;
-
-        MergePathSearch(
-            OffsetT(threadIdx.x * ITEMS_PER_THREAD),    // Diagonal
-            s_tile_row_end_offsets,                     // List A
-            tile_nonzero_indices,                       // List B
-            tile_num_rows,
-            tile_num_nonzeros,
-            thread_start_coord);
-
-        CTA_SYNC();            // Perf-sync
-
-        // Compute the thread's merge path segment
-        CoordinateT     thread_current_coord = thread_start_coord;
-        KeyValuePairT   scan_segment[ITEMS_PER_THREAD];
-
-        ValueT          running_total = 0.0;
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            OffsetT nonzero_idx         = CUB_MIN(tile_nonzero_indices[thread_current_coord.y], spmv_params.num_nonzeros - 1);
-            OffsetT column_idx          = wd_column_indices[nonzero_idx];
-            ValueT  value               = wd_values[nonzero_idx];
-
-            ValueT  vector_value        = spmv_params.t_vector_x[column_idx];
-#if (CUB_PTX_ARCH >= 350)
-            vector_value                = wd_vector_x[column_idx];
-#endif
-            ValueT  nonzero             = value * vector_value;
-
-            OffsetT row_end_offset      = s_tile_row_end_offsets[thread_current_coord.x];
-
-            if (tile_nonzero_indices[thread_current_coord.y] < row_end_offset)
-            {
-                // Move down (accumulate)
-                running_total += nonzero;
-                scan_segment[ITEM].value    = running_total;
-                scan_segment[ITEM].key      = tile_num_rows;
-                ++thread_current_coord.y;
-            }
-            else
-            {
-                // Move right (reset)
-                scan_segment[ITEM].value    = running_total;
-                scan_segment[ITEM].key      = thread_current_coord.x;
-                running_total               = 0.0;
-                ++thread_current_coord.x;
-            }
-        }
-
-        CTA_SYNC();
-
-        // Block-wide reduce-value-by-segment
-        KeyValuePairT       tile_carry;
-        ReduceBySegmentOpT  scan_op;
-        KeyValuePairT       scan_item;
-
-        scan_item.value = running_total;
-        scan_item.key   = thread_current_coord.x;
-
-        BlockScanT(temp_storage.aliasable.scan).ExclusiveScan(scan_item, scan_item, scan_op, tile_carry);
-
-        if (tile_num_rows > 0)
-        {
-            if (threadIdx.x == 0)
-                scan_item.key = -1;
-
-            // Direct scatter
-            #pragma unroll
-            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-            {
-                if (scan_segment[ITEM].key < tile_num_rows)
-                {
-                    if (scan_item.key == scan_segment[ITEM].key)
-                        scan_segment[ITEM].value = scan_item.value + scan_segment[ITEM].value;
-
-                    if (HAS_ALPHA)
-                    {
-                        scan_segment[ITEM].value *= spmv_params.alpha;
-                    }
-
-                    if (HAS_BETA)
-                    {
-                        // Update the output vector element
-                        ValueT addend = spmv_params.beta * wd_vector_y[tile_start_coord.x + scan_segment[ITEM].key];
-                        scan_segment[ITEM].value += addend;
-                    }
-
-                    // Set the output vector element
-                    spmv_params.d_vector_y[tile_start_coord.x + scan_segment[ITEM].key] = scan_segment[ITEM].value;
-                }
-            }
-        }
-
-        // Return the tile's running carry-out
-        return tile_carry;
-    }
-
-
-
-    /**
-     * Consume a merge tile, specialized for indirect load of nonzeros
-     */
-    __device__ __forceinline__ KeyValuePairT ConsumeTile(
-        int             tile_idx,
-        CoordinateT     tile_start_coord,
-        CoordinateT     tile_end_coord,
-        Int2Type<false> is_direct_load)     ///< Marker type indicating whether to load nonzeros directly during path-discovery or beforehand in batch
-    {
-        int         tile_num_rows           = tile_end_coord.x - tile_start_coord.x;
-        int         tile_num_nonzeros       = tile_end_coord.y - tile_start_coord.y;
-
-#if (CUB_PTX_ARCH >= 520)
-
-        OffsetT*    s_tile_row_end_offsets  = &temp_storage.aliasable.merge_items[0].row_end_offset;
-        ValueT*     s_tile_nonzeros         = &temp_storage.aliasable.merge_items[tile_num_rows + ITEMS_PER_THREAD].nonzero;
-
-        // Gather the nonzeros for the merge tile into shared memory
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            int nonzero_idx = threadIdx.x + (ITEM * BLOCK_THREADS);
-
-            ValueIteratorT a                = wd_values + tile_start_coord.y + nonzero_idx;
-            ColumnIndicesIteratorT ci       = wd_column_indices + tile_start_coord.y + nonzero_idx;
-            ValueT* s                       = s_tile_nonzeros + nonzero_idx;
-
-            if (nonzero_idx < tile_num_nonzeros)
-            {
-
-                OffsetT column_idx              = *ci;
-                ValueT  value                   = *a;
-
-                ValueT  vector_value            = spmv_params.t_vector_x[column_idx];
-                vector_value                    = wd_vector_x[column_idx];
-
-                ValueT  nonzero                 = value * vector_value;
-
-                *s    = nonzero;
-            }
-        }
-
-
-#else
-
-        OffsetT*    s_tile_row_end_offsets  = &temp_storage.aliasable.merge_items[0].row_end_offset;
-        ValueT*     s_tile_nonzeros         = &temp_storage.aliasable.merge_items[tile_num_rows + ITEMS_PER_THREAD].nonzero;
-
-        // Gather the nonzeros for the merge tile into shared memory
-        if (tile_num_nonzeros > 0)
-        {
-            #pragma unroll
-            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-            {
-                int     nonzero_idx             = threadIdx.x + (ITEM * BLOCK_THREADS);
-                nonzero_idx                     = CUB_MIN(nonzero_idx, tile_num_nonzeros - 1);
-
-                OffsetT column_idx              = wd_column_indices[tile_start_coord.y + nonzero_idx];
-                ValueT  value                   = wd_values[tile_start_coord.y + nonzero_idx];
-
-                ValueT  vector_value            = spmv_params.t_vector_x[column_idx];
-#if (CUB_PTX_ARCH >= 350)
-                vector_value                    = wd_vector_x[column_idx];
-#endif
-                ValueT  nonzero                 = value * vector_value;
-
-                s_tile_nonzeros[nonzero_idx]    = nonzero;
-            }
-        }
-
-#endif
-
-        // Gather the row end-offsets for the merge tile into shared memory
-        #pragma unroll 1
-        for (int item = threadIdx.x; item <= tile_num_rows; item += BLOCK_THREADS)
-        {
-            s_tile_row_end_offsets[item] = wd_row_end_offsets[tile_start_coord.x + item];
-        }
-
-        CTA_SYNC();
-
-        // Search for the thread's starting coordinate within the merge tile
-        CountingInputIterator<OffsetT>  tile_nonzero_indices(tile_start_coord.y);
-        CoordinateT                     thread_start_coord;
-
-        MergePathSearch(
-            OffsetT(threadIdx.x * ITEMS_PER_THREAD),    // Diagonal
-            s_tile_row_end_offsets,                     // List A
-            tile_nonzero_indices,                       // List B
-            tile_num_rows,
-            tile_num_nonzeros,
-            thread_start_coord);
-
-        CTA_SYNC();            // Perf-sync
-
-        // Compute the thread's merge path segment
-        CoordinateT     thread_current_coord = thread_start_coord;
-        KeyValuePairT   scan_segment[ITEMS_PER_THREAD];
-        ValueT          running_total = 0.0;
-
-        OffsetT row_end_offset  = s_tile_row_end_offsets[thread_current_coord.x];
-        ValueT  nonzero         = s_tile_nonzeros[thread_current_coord.y];
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-        {
-            if (tile_nonzero_indices[thread_current_coord.y] < row_end_offset)
-            {
-                // Move down (accumulate)
-                scan_segment[ITEM].value    = nonzero;
-                running_total               += nonzero;
-                ++thread_current_coord.y;
-                nonzero                     = s_tile_nonzeros[thread_current_coord.y];
-            }
-            else
-            {
-                // Move right (reset)
-                scan_segment[ITEM].value    = 0.0;
-                running_total               = 0.0;
-                ++thread_current_coord.x;
-                row_end_offset              = s_tile_row_end_offsets[thread_current_coord.x];
-            }
-
-            scan_segment[ITEM].key = thread_current_coord.x;
-        }
-
-        CTA_SYNC();
-
-        // Block-wide reduce-value-by-segment
-        KeyValuePairT       tile_carry;
-        ReduceBySegmentOpT  scan_op;
-        KeyValuePairT       scan_item;
-
-        scan_item.value = running_total;
-        scan_item.key = thread_current_coord.x;
-
-        BlockScanT(temp_storage.aliasable.scan).ExclusiveScan(scan_item, scan_item, scan_op, tile_carry);
-
-        if (threadIdx.x == 0)
-        {
-            scan_item.key = thread_start_coord.x;
-            scan_item.value = 0.0;
-        }
-
-        if (tile_num_rows > 0)
-        {
-
-            CTA_SYNC();
-
-            // Scan downsweep and scatter
-            ValueT* s_partials = &temp_storage.aliasable.merge_items[0].nonzero;
-
-            if (scan_item.key != scan_segment[0].key)
-            {
-                s_partials[scan_item.key] = scan_item.value;
-            }
-            else
-            {
-                scan_segment[0].value += scan_item.value;
-            }
-
-            #pragma unroll
-            for (int ITEM = 1; ITEM < ITEMS_PER_THREAD; ++ITEM)
-            {
-                if (scan_segment[ITEM - 1].key != scan_segment[ITEM].key)
-                {
-                    s_partials[scan_segment[ITEM - 1].key] = scan_segment[ITEM - 1].value;
-                }
-                else
-                {
-                    scan_segment[ITEM].value += scan_segment[ITEM - 1].value;
-                }
-            }
-
-            CTA_SYNC();
-
-            #pragma unroll 1
-            for (int item = threadIdx.x; item < tile_num_rows; item += BLOCK_THREADS)
-            {
-                spmv_params.d_vector_y[tile_start_coord.x + item] = s_partials[item];
-            }
-        }
-
-        // Return the tile's running carry-out
-        return tile_carry;
-    }
-
-
-    /**
-     * Consume input tile
-     */
-    __device__ __forceinline__ void ConsumeTile(
-        CoordinateT*    d_tile_coordinates,     ///< [in] Pointer to the temporary array of tile starting coordinates
-        KeyValuePairT*  d_tile_carry_pairs,     ///< [out] Pointer to the temporary array carry-out dot product row-ids, one per block
-        int             num_merge_tiles)        ///< [in] Number of merge tiles
-    {
-        int tile_idx = (blockIdx.x * gridDim.y) + blockIdx.y;    // Current tile index
-
-        if (tile_idx >= num_merge_tiles)
-            return;
-
-        // Read our starting coordinates
-        if (threadIdx.x < 2)
-        {
-            if (d_tile_coordinates == NULL)
-            {
-                // Search our starting coordinates
-                OffsetT                         diagonal = (tile_idx + threadIdx.x) * TILE_ITEMS;
-                CoordinateT                     tile_coord;
-                CountingInputIterator<OffsetT>  nonzero_indices(0);
-
-                // Search the merge path
-                MergePathSearch(
-                    diagonal,
-                    RowOffsetsSearchIteratorT(spmv_params.d_row_end_offsets),
-                    nonzero_indices,
-                    spmv_params.num_rows,
-                    spmv_params.num_nonzeros,
-                    tile_coord);
-
-                temp_storage.tile_coords[threadIdx.x] = tile_coord;
-            }
-            else
-            {
-                temp_storage.tile_coords[threadIdx.x] = d_tile_coordinates[tile_idx + threadIdx.x];
-            }
-        }
-
-        CTA_SYNC();
-
-        CoordinateT tile_start_coord     = temp_storage.tile_coords[0];
-        CoordinateT tile_end_coord       = temp_storage.tile_coords[1];
-
-        // Consume multi-segment tile
-        KeyValuePairT tile_carry = ConsumeTile(
-            tile_idx,
-            tile_start_coord,
-            tile_end_coord,
-            Int2Type<AgentSpmvPolicyT::DIRECT_LOAD_NONZEROS>());
-
-        // Output the tile's carry-out
-        if (threadIdx.x == 0)
-        {
-            if (HAS_ALPHA)
-                tile_carry.value *= spmv_params.alpha;
-
-            tile_carry.key += tile_start_coord.x;
-            d_tile_carry_pairs[tile_idx]    = tile_carry;
-        }
-    }
-
-
-};
-
-
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/agent/single_pass_scan_operators.cuh b/pfsimulator/third_party/cub/agent/single_pass_scan_operators.cuh
deleted file mode 100644
index 53409bdee..000000000
--- a/pfsimulator/third_party/cub/agent/single_pass_scan_operators.cuh
+++ /dev/null
@@ -1,815 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Callback operator types for supplying BlockScan prefixes
- */
-
-#pragma once
-
-#include <iterator>
-
-#include "../thread/thread_load.cuh"
-#include "../thread/thread_store.cuh"
-#include "../warp/warp_reduce.cuh"
-#include "../util_arch.cuh"
-#include "../util_device.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/******************************************************************************
- * Prefix functor type for maintaining a running prefix while scanning a
- * region independent of other thread blocks
- ******************************************************************************/
-
-/**
- * Stateful callback operator type for supplying BlockScan prefixes.
- * Maintains a running prefix that can be applied to consecutive
- * BlockScan operations.
- */
-template <
-    typename T,                 ///< BlockScan value type
-    typename ScanOpT>            ///< Wrapped scan operator type
-struct BlockScanRunningPrefixOp
-{
-    ScanOpT     op;                 ///< Wrapped scan operator
-    T           running_total;      ///< Running block-wide prefix
-
-    /// Constructor
-    __device__ __forceinline__ BlockScanRunningPrefixOp(ScanOpT op)
-    :
-        op(op)
-    {}
-
-    /// Constructor
-    __device__ __forceinline__ BlockScanRunningPrefixOp(
-        T starting_prefix,
-        ScanOpT op)
-    :
-        op(op),
-        running_total(starting_prefix)
-    {}
-
-    /**
-     * Prefix callback operator.  Returns the block-wide running_total in thread-0.
-     */
-    __device__ __forceinline__ T operator()(
-        const T &block_aggregate)              ///< The aggregate sum of the BlockScan inputs
-    {
-        T retval = running_total;
-        running_total = op(running_total, block_aggregate);
-        return retval;
-    }
-};
-
-
-/******************************************************************************
- * Generic tile status interface types for block-cooperative scans
- ******************************************************************************/
-
-/**
- * Enumerations of tile status
- */
-enum ScanTileStatus
-{
-    SCAN_TILE_OOB,          // Out-of-bounds (e.g., padding)
-    SCAN_TILE_INVALID = 99, // Not yet processed
-    SCAN_TILE_PARTIAL,      // Tile aggregate is available
-    SCAN_TILE_INCLUSIVE,    // Inclusive tile prefix is available
-};
-
-
-/**
- * Tile status interface.
- */
-template <
-    typename    T,
-    bool        SINGLE_WORD = Traits<T>::PRIMITIVE>
-struct ScanTileState;
-
-
-/**
- * Tile status interface specialized for scan status and value types
- * that can be combined into one machine word that can be
- * read/written coherently in a single access.
- */
-template <typename T>
-struct ScanTileState<T, true>
-{
-    // Status word type
-    typedef typename If<(sizeof(T) == 8),
-        long long,
-        typename If<(sizeof(T) == 4),
-            int,
-            typename If<(sizeof(T) == 2),
-                short,
-                char>::Type>::Type>::Type StatusWord;
-
-
-    // Unit word type
-    typedef typename If<(sizeof(T) == 8),
-        longlong2,
-        typename If<(sizeof(T) == 4),
-            int2,
-            typename If<(sizeof(T) == 2),
-                int,
-                uchar2>::Type>::Type>::Type TxnWord;
-
-
-    // Device word type
-    struct TileDescriptor
-    {
-        StatusWord  status;
-        T           value;
-    };
-
-
-    // Constants
-    enum
-    {
-        TILE_STATUS_PADDING = CUB_PTX_WARP_THREADS,
-    };
-
-
-    // Device storage
-    TxnWord *d_tile_descriptors;
-
-    /// Constructor
-    __host__ __device__ __forceinline__
-    ScanTileState()
-    :
-        d_tile_descriptors(NULL)
-    {}
-
-
-    /// Initializer
-    __host__ __device__ __forceinline__
-    cudaError_t Init(
-        int     /*num_tiles*/,                      ///< [in] Number of tiles
-        void    *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t  /*temp_storage_bytes*/)             ///< [in] Size in bytes of \t d_temp_storage allocation
-    {
-        d_tile_descriptors = reinterpret_cast<TxnWord*>(d_temp_storage);
-        return cudaSuccess;
-    }
-
-
-    /**
-     * Compute device memory needed for tile status
-     */
-    __host__ __device__ __forceinline__
-    static cudaError_t AllocationSize(
-        int     num_tiles,                          ///< [in] Number of tiles
-        size_t  &temp_storage_bytes)                ///< [out] Size in bytes of \t d_temp_storage allocation
-    {
-        temp_storage_bytes = (num_tiles + TILE_STATUS_PADDING) * sizeof(TileDescriptor);       // bytes needed for tile status descriptors
-        return cudaSuccess;
-    }
-
-
-    /**
-     * Initialize (from device)
-     */
-    __device__ __forceinline__ void InitializeStatus(int num_tiles)
-    {
-        int tile_idx = (blockIdx.x * blockDim.x) + threadIdx.x;
-
-        TxnWord val = TxnWord();
-        TileDescriptor *descriptor = reinterpret_cast<TileDescriptor*>(&val);
-
-        if (tile_idx < num_tiles)
-        {
-            // Not-yet-set
-            descriptor->status = StatusWord(SCAN_TILE_INVALID);
-            d_tile_descriptors[TILE_STATUS_PADDING + tile_idx] = val;
-        }
-
-        if ((blockIdx.x == 0) && (threadIdx.x < TILE_STATUS_PADDING))
-        {
-            // Padding
-            descriptor->status = StatusWord(SCAN_TILE_OOB);
-            d_tile_descriptors[threadIdx.x] = val;
-        }
-    }
-
-
-    /**
-     * Update the specified tile's inclusive value and corresponding status
-     */
-    __device__ __forceinline__ void SetInclusive(int tile_idx, T tile_inclusive)
-    {
-        TileDescriptor tile_descriptor;
-        tile_descriptor.status = SCAN_TILE_INCLUSIVE;
-        tile_descriptor.value = tile_inclusive;
-
-        TxnWord alias;
-        *reinterpret_cast<TileDescriptor*>(&alias) = tile_descriptor;
-        ThreadStore<STORE_CG>(d_tile_descriptors + TILE_STATUS_PADDING + tile_idx, alias);
-    }
-
-
-    /**
-     * Update the specified tile's partial value and corresponding status
-     */
-    __device__ __forceinline__ void SetPartial(int tile_idx, T tile_partial)
-    {
-        TileDescriptor tile_descriptor;
-        tile_descriptor.status = SCAN_TILE_PARTIAL;
-        tile_descriptor.value = tile_partial;
-
-        TxnWord alias;
-        *reinterpret_cast<TileDescriptor*>(&alias) = tile_descriptor;
-        ThreadStore<STORE_CG>(d_tile_descriptors + TILE_STATUS_PADDING + tile_idx, alias);
-    }
-
-    /**
-     * Wait for the corresponding tile to become non-invalid
-     */
-    __device__ __forceinline__ void WaitForValid(
-        int             tile_idx,
-        StatusWord      &status,
-        T               &value)
-    {
-        TileDescriptor tile_descriptor;
-        do
-        {
-            __threadfence_block(); // prevent hoisting loads from loop
-            TxnWord alias = ThreadLoad<LOAD_CG>(d_tile_descriptors + TILE_STATUS_PADDING + tile_idx);
-            tile_descriptor = reinterpret_cast<TileDescriptor&>(alias);
-
-        } while (WARP_ANY((tile_descriptor.status == SCAN_TILE_INVALID), 0xffffffff));
-
-        status = tile_descriptor.status;
-        value = tile_descriptor.value;
-    }
-
-};
-
-
-
-/**
- * Tile status interface specialized for scan status and value types that
- * cannot be combined into one machine word.
- */
-template <typename T>
-struct ScanTileState<T, false>
-{
-    // Status word type
-    typedef char StatusWord;
-
-    // Constants
-    enum
-    {
-        TILE_STATUS_PADDING = CUB_PTX_WARP_THREADS,
-    };
-
-    // Device storage
-    StatusWord  *d_tile_status;
-    T           *d_tile_partial;
-    T           *d_tile_inclusive;
-
-    /// Constructor
-    __host__ __device__ __forceinline__
-    ScanTileState()
-    :
-        d_tile_status(NULL),
-        d_tile_partial(NULL),
-        d_tile_inclusive(NULL)
-    {}
-
-
-    /// Initializer
-    __host__ __device__ __forceinline__
-    cudaError_t Init(
-        int     num_tiles,                          ///< [in] Number of tiles
-        void    *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t  temp_storage_bytes)                 ///< [in] Size in bytes of \t d_temp_storage allocation
-    {
-        cudaError_t error = cudaSuccess;
-        do
-        {
-            void*   allocations[3];
-            size_t  allocation_sizes[3];
-
-            allocation_sizes[0] = (num_tiles + TILE_STATUS_PADDING) * sizeof(StatusWord);           // bytes needed for tile status descriptors
-            allocation_sizes[1] = (num_tiles + TILE_STATUS_PADDING) * sizeof(Uninitialized<T>);     // bytes needed for partials
-            allocation_sizes[2] = (num_tiles + TILE_STATUS_PADDING) * sizeof(Uninitialized<T>);     // bytes needed for inclusives
-
-            // Compute allocation pointers into the single storage blob
-            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
-
-            // Alias the offsets
-            d_tile_status       = reinterpret_cast<StatusWord*>(allocations[0]);
-            d_tile_partial      = reinterpret_cast<T*>(allocations[1]);
-            d_tile_inclusive    = reinterpret_cast<T*>(allocations[2]);
-        }
-        while (0);
-
-        return error;
-    }
-
-
-    /**
-     * Compute device memory needed for tile status
-     */
-    __host__ __device__ __forceinline__
-    static cudaError_t AllocationSize(
-        int     num_tiles,                          ///< [in] Number of tiles
-        size_t  &temp_storage_bytes)                ///< [out] Size in bytes of \t d_temp_storage allocation
-    {
-        // Specify storage allocation requirements
-        size_t  allocation_sizes[3];
-        allocation_sizes[0] = (num_tiles + TILE_STATUS_PADDING) * sizeof(StatusWord);         // bytes needed for tile status descriptors
-        allocation_sizes[1] = (num_tiles + TILE_STATUS_PADDING) * sizeof(Uninitialized<T>);   // bytes needed for partials
-        allocation_sizes[2] = (num_tiles + TILE_STATUS_PADDING) * sizeof(Uninitialized<T>);   // bytes needed for inclusives
-
-        // Set the necessary size of the blob
-        void* allocations[3];
-        return CubDebug(AliasTemporaries(NULL, temp_storage_bytes, allocations, allocation_sizes));
-    }
-
-
-    /**
-     * Initialize (from device)
-     */
-    __device__ __forceinline__ void InitializeStatus(int num_tiles)
-    {
-        int tile_idx = (blockIdx.x * blockDim.x) + threadIdx.x;
-        if (tile_idx < num_tiles)
-        {
-            // Not-yet-set
-            d_tile_status[TILE_STATUS_PADDING + tile_idx] = StatusWord(SCAN_TILE_INVALID);
-        }
-
-        if ((blockIdx.x == 0) && (threadIdx.x < TILE_STATUS_PADDING))
-        {
-            // Padding
-            d_tile_status[threadIdx.x] = StatusWord(SCAN_TILE_OOB);
-        }
-    }
-
-
-    /**
-     * Update the specified tile's inclusive value and corresponding status
-     */
-    __device__ __forceinline__ void SetInclusive(int tile_idx, T tile_inclusive)
-    {
-        // Update tile inclusive value
-        ThreadStore<STORE_CG>(d_tile_inclusive + TILE_STATUS_PADDING + tile_idx, tile_inclusive);
-
-        // Fence
-        __threadfence();
-
-        // Update tile status
-        ThreadStore<STORE_CG>(d_tile_status + TILE_STATUS_PADDING + tile_idx, StatusWord(SCAN_TILE_INCLUSIVE));
-    }
-
-
-    /**
-     * Update the specified tile's partial value and corresponding status
-     */
-    __device__ __forceinline__ void SetPartial(int tile_idx, T tile_partial)
-    {
-        // Update tile partial value
-        ThreadStore<STORE_CG>(d_tile_partial + TILE_STATUS_PADDING + tile_idx, tile_partial);
-
-        // Fence
-        __threadfence();
-
-        // Update tile status
-        ThreadStore<STORE_CG>(d_tile_status + TILE_STATUS_PADDING + tile_idx, StatusWord(SCAN_TILE_PARTIAL));
-    }
-
-    /**
-     * Wait for the corresponding tile to become non-invalid
-     */
-    __device__ __forceinline__ void WaitForValid(
-        int             tile_idx,
-        StatusWord      &status,
-        T               &value)
-    {
-        do {
-            status = ThreadLoad<LOAD_CG>(d_tile_status + TILE_STATUS_PADDING + tile_idx);
-
-            __threadfence();    // prevent hoisting loads from loop or loads below above this one
-
-        } while (status == SCAN_TILE_INVALID);
-
-        if (status == StatusWord(SCAN_TILE_PARTIAL)) 
-            value = ThreadLoad<LOAD_CG>(d_tile_partial + TILE_STATUS_PADDING + tile_idx);
-        else
-            value = ThreadLoad<LOAD_CG>(d_tile_inclusive + TILE_STATUS_PADDING + tile_idx);
-    }
-};
-
-
-/******************************************************************************
- * ReduceByKey tile status interface types for block-cooperative scans
- ******************************************************************************/
-
-/**
- * Tile status interface for reduction by key.
- *
- */
-template <
-    typename    ValueT,
-    typename    KeyT,
-    bool        SINGLE_WORD = (Traits<ValueT>::PRIMITIVE) && (sizeof(ValueT) + sizeof(KeyT) < 16)>
-struct ReduceByKeyScanTileState;
-
-
-/**
- * Tile status interface for reduction by key, specialized for scan status and value types that
- * cannot be combined into one machine word.
- */
-template <
-    typename    ValueT,
-    typename    KeyT>
-struct ReduceByKeyScanTileState<ValueT, KeyT, false> :
-    ScanTileState<KeyValuePair<KeyT, ValueT> >
-{
-    typedef ScanTileState<KeyValuePair<KeyT, ValueT> > SuperClass;
-
-    /// Constructor
-    __host__ __device__ __forceinline__
-    ReduceByKeyScanTileState() : SuperClass() {}
-};
-
-
-/**
- * Tile status interface for reduction by key, specialized for scan status and value types that
- * can be combined into one machine word that can be read/written coherently in a single access.
- */
-template <
-    typename ValueT,
-    typename KeyT>
-struct ReduceByKeyScanTileState<ValueT, KeyT, true>
-{
-    typedef KeyValuePair<KeyT, ValueT>KeyValuePairT;
-
-    // Constants
-    enum
-    {
-        PAIR_SIZE           = sizeof(ValueT) + sizeof(KeyT),
-        TXN_WORD_SIZE       = 1 << Log2<PAIR_SIZE + 1>::VALUE,
-        STATUS_WORD_SIZE    = TXN_WORD_SIZE - PAIR_SIZE,
-
-        TILE_STATUS_PADDING = CUB_PTX_WARP_THREADS,
-    };
-
-    // Status word type
-    typedef typename If<(STATUS_WORD_SIZE == 8),
-        long long,
-        typename If<(STATUS_WORD_SIZE == 4),
-            int,
-            typename If<(STATUS_WORD_SIZE == 2),
-                short,
-                char>::Type>::Type>::Type StatusWord;
-
-    // Status word type
-    typedef typename If<(TXN_WORD_SIZE == 16),
-        longlong2,
-        typename If<(TXN_WORD_SIZE == 8),
-            long long,
-            int>::Type>::Type TxnWord;
-
-    // Device word type (for when sizeof(ValueT) == sizeof(KeyT))
-    struct TileDescriptorBigStatus
-    {
-        KeyT        key;
-        ValueT      value;
-        StatusWord  status;
-    };
-
-    // Device word type (for when sizeof(ValueT) != sizeof(KeyT))
-    struct TileDescriptorLittleStatus
-    {
-        ValueT      value;
-        StatusWord  status;
-        KeyT        key;
-    };
-
-    // Device word type
-    typedef typename If<
-            (sizeof(ValueT) == sizeof(KeyT)),
-            TileDescriptorBigStatus,
-            TileDescriptorLittleStatus>::Type
-        TileDescriptor;
-
-
-    // Device storage
-    TxnWord *d_tile_descriptors;
-
-
-    /// Constructor
-    __host__ __device__ __forceinline__
-    ReduceByKeyScanTileState()
-    :
-        d_tile_descriptors(NULL)
-    {}
-
-
-    /// Initializer
-    __host__ __device__ __forceinline__
-    cudaError_t Init(
-        int     /*num_tiles*/,                      ///< [in] Number of tiles
-        void    *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t  /*temp_storage_bytes*/)             ///< [in] Size in bytes of \t d_temp_storage allocation
-    {
-        d_tile_descriptors = reinterpret_cast<TxnWord*>(d_temp_storage);
-        return cudaSuccess;
-    }
-
-
-    /**
-     * Compute device memory needed for tile status
-     */
-    __host__ __device__ __forceinline__
-    static cudaError_t AllocationSize(
-        int     num_tiles,                          ///< [in] Number of tiles
-        size_t  &temp_storage_bytes)                ///< [out] Size in bytes of \t d_temp_storage allocation
-    {
-        temp_storage_bytes = (num_tiles + TILE_STATUS_PADDING) * sizeof(TileDescriptor);       // bytes needed for tile status descriptors
-        return cudaSuccess;
-    }
-
-
-    /**
-     * Initialize (from device)
-     */
-    __device__ __forceinline__ void InitializeStatus(int num_tiles)
-    {
-        int             tile_idx    = (blockIdx.x * blockDim.x) + threadIdx.x;
-        TxnWord         val         = TxnWord();
-        TileDescriptor  *descriptor = reinterpret_cast<TileDescriptor*>(&val);
-
-        if (tile_idx < num_tiles)
-        {
-            // Not-yet-set
-            descriptor->status = StatusWord(SCAN_TILE_INVALID);
-            d_tile_descriptors[TILE_STATUS_PADDING + tile_idx] = val;
-        }
-
-        if ((blockIdx.x == 0) && (threadIdx.x < TILE_STATUS_PADDING))
-        {
-            // Padding
-            descriptor->status = StatusWord(SCAN_TILE_OOB);
-            d_tile_descriptors[threadIdx.x] = val;
-        }
-    }
-
-
-    /**
-     * Update the specified tile's inclusive value and corresponding status
-     */
-    __device__ __forceinline__ void SetInclusive(int tile_idx, KeyValuePairT tile_inclusive)
-    {
-        TileDescriptor tile_descriptor;
-        tile_descriptor.status  = SCAN_TILE_INCLUSIVE;
-        tile_descriptor.value   = tile_inclusive.value;
-        tile_descriptor.key     = tile_inclusive.key;
-
-        TxnWord alias;
-        *reinterpret_cast<TileDescriptor*>(&alias) = tile_descriptor;
-        ThreadStore<STORE_CG>(d_tile_descriptors + TILE_STATUS_PADDING + tile_idx, alias);
-    }
-
-
-    /**
-     * Update the specified tile's partial value and corresponding status
-     */
-    __device__ __forceinline__ void SetPartial(int tile_idx, KeyValuePairT tile_partial)
-    {
-        TileDescriptor tile_descriptor;
-        tile_descriptor.status  = SCAN_TILE_PARTIAL;
-        tile_descriptor.value   = tile_partial.value;
-        tile_descriptor.key     = tile_partial.key;
-
-        TxnWord alias;
-        *reinterpret_cast<TileDescriptor*>(&alias) = tile_descriptor;
-        ThreadStore<STORE_CG>(d_tile_descriptors + TILE_STATUS_PADDING + tile_idx, alias);
-    }
-
-    /**
-     * Wait for the corresponding tile to become non-invalid
-     */
-    __device__ __forceinline__ void WaitForValid(
-        int                     tile_idx,
-        StatusWord              &status,
-        KeyValuePairT           &value)
-    {
-//        TxnWord         alias           = ThreadLoad<LOAD_CG>(d_tile_descriptors + TILE_STATUS_PADDING + tile_idx);
-//        TileDescriptor  tile_descriptor = reinterpret_cast<TileDescriptor&>(alias);
-//
-//        while (tile_descriptor.status == SCAN_TILE_INVALID)
-//        {
-//            __threadfence_block(); // prevent hoisting loads from loop
-//
-//            alias           = ThreadLoad<LOAD_CG>(d_tile_descriptors + TILE_STATUS_PADDING + tile_idx);
-//            tile_descriptor = reinterpret_cast<TileDescriptor&>(alias);
-//        }
-//
-//        status      = tile_descriptor.status;
-//        value.value = tile_descriptor.value;
-//        value.key   = tile_descriptor.key;
-
-        TileDescriptor tile_descriptor;
-        do
-        {
-            __threadfence_block(); // prevent hoisting loads from loop
-            TxnWord alias = ThreadLoad<LOAD_CG>(d_tile_descriptors + TILE_STATUS_PADDING + tile_idx);
-            tile_descriptor = reinterpret_cast<TileDescriptor&>(alias);
-
-        } while (WARP_ANY((tile_descriptor.status == SCAN_TILE_INVALID), 0xffffffff));
-
-        status      = tile_descriptor.status;
-        value.value = tile_descriptor.value;
-        value.key   = tile_descriptor.key;
-    }
-
-};
-
-
-/******************************************************************************
- * Prefix call-back operator for coupling local block scan within a
- * block-cooperative scan
- ******************************************************************************/
-
-/**
- * Stateful block-scan prefix functor.  Provides the the running prefix for
- * the current tile by using the call-back warp to wait on on
- * aggregates/prefixes from predecessor tiles to become available.
- */
-template <
-    typename    T,
-    typename    ScanOpT,
-    typename    ScanTileStateT,
-    int         PTX_ARCH = CUB_PTX_ARCH>
-struct TilePrefixCallbackOp
-{
-    // Parameterized warp reduce
-    typedef WarpReduce<T, CUB_PTX_WARP_THREADS, PTX_ARCH> WarpReduceT;
-
-    // Temporary storage type
-    struct _TempStorage
-    {
-        typename WarpReduceT::TempStorage   warp_reduce;
-        T                                   exclusive_prefix;
-        T                                   inclusive_prefix;
-        T                                   block_aggregate;
-    };
-
-    // Alias wrapper allowing temporary storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-    // Type of status word
-    typedef typename ScanTileStateT::StatusWord StatusWord;
-
-    // Fields
-    _TempStorage&               temp_storage;       ///< Reference to a warp-reduction instance
-    ScanTileStateT&             tile_status;        ///< Interface to tile status
-    ScanOpT                     scan_op;            ///< Binary scan operator
-    int                         tile_idx;           ///< The current tile index
-    T                           exclusive_prefix;   ///< Exclusive prefix for the tile
-    T                           inclusive_prefix;   ///< Inclusive prefix for the tile
-
-    // Constructor
-    __device__ __forceinline__
-    TilePrefixCallbackOp(
-        ScanTileStateT       &tile_status,
-        TempStorage         &temp_storage,
-        ScanOpT              scan_op,
-        int                 tile_idx)
-    :
-        temp_storage(temp_storage.Alias()),
-        tile_status(tile_status),
-        scan_op(scan_op),
-        tile_idx(tile_idx) {}
-
-
-    // Block until all predecessors within the warp-wide window have non-invalid status
-    __device__ __forceinline__
-    void ProcessWindow(
-        int         predecessor_idx,        ///< Preceding tile index to inspect
-        StatusWord  &predecessor_status,    ///< [out] Preceding tile status
-        T           &window_aggregate)      ///< [out] Relevant partial reduction from this window of preceding tiles
-    {
-        T value;
-        tile_status.WaitForValid(predecessor_idx, predecessor_status, value);
-
-        // Perform a segmented reduction to get the prefix for the current window.
-        // Use the swizzled scan operator because we are now scanning *down* towards thread0.
-
-        int tail_flag = (predecessor_status == StatusWord(SCAN_TILE_INCLUSIVE));
-        window_aggregate = WarpReduceT(temp_storage.warp_reduce).TailSegmentedReduce(
-            value,
-            tail_flag,
-            SwizzleScanOp<ScanOpT>(scan_op));
-    }
-
-
-    // BlockScan prefix callback functor (called by the first warp)
-    __device__ __forceinline__
-    T operator()(T block_aggregate)
-    {
-
-        // Update our status with our tile-aggregate
-        if (threadIdx.x == 0)
-        {
-            temp_storage.block_aggregate = block_aggregate;
-            tile_status.SetPartial(tile_idx, block_aggregate);
-        }
-
-        int         predecessor_idx = tile_idx - threadIdx.x - 1;
-        StatusWord  predecessor_status;
-        T           window_aggregate;
-
-        // Wait for the warp-wide window of predecessor tiles to become valid
-        ProcessWindow(predecessor_idx, predecessor_status, window_aggregate);
-
-        // The exclusive tile prefix starts out as the current window aggregate
-        exclusive_prefix = window_aggregate;
-
-        // Keep sliding the window back until we come across a tile whose inclusive prefix is known
-        while (WARP_ALL((predecessor_status != StatusWord(SCAN_TILE_INCLUSIVE)), 0xffffffff))
-        {
-            predecessor_idx -= CUB_PTX_WARP_THREADS;
-
-            // Update exclusive tile prefix with the window prefix
-            ProcessWindow(predecessor_idx, predecessor_status, window_aggregate);
-            exclusive_prefix = scan_op(window_aggregate, exclusive_prefix);
-        }
-
-        // Compute the inclusive tile prefix and update the status for this tile
-        if (threadIdx.x == 0)
-        {
-            inclusive_prefix = scan_op(exclusive_prefix, block_aggregate);
-            tile_status.SetInclusive(tile_idx, inclusive_prefix);
-
-            temp_storage.exclusive_prefix = exclusive_prefix;
-            temp_storage.inclusive_prefix = inclusive_prefix;
-        }
-
-        // Return exclusive_prefix
-        return exclusive_prefix;
-    }
-
-    // Get the exclusive prefix stored in temporary storage
-    __device__ __forceinline__
-    T GetExclusivePrefix()
-    {
-        return temp_storage.exclusive_prefix;
-    }
-
-    // Get the inclusive prefix stored in temporary storage
-    __device__ __forceinline__
-    T GetInclusivePrefix()
-    {
-        return temp_storage.inclusive_prefix;
-    }
-
-    // Get the block aggregate stored in temporary storage
-    __device__ __forceinline__
-    T GetBlockAggregate()
-    {
-        return temp_storage.block_aggregate;
-    }
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/block/block_adjacent_difference.cuh b/pfsimulator/third_party/cub/block/block_adjacent_difference.cuh
deleted file mode 100644
index acef9f056..000000000
--- a/pfsimulator/third_party/cub/block/block_adjacent_difference.cuh
+++ /dev/null
@@ -1,596 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * The cub::BlockDiscontinuity class provides [<em>collective</em>](index.html#sec0) methods for flagging discontinuities within an ordered set of items partitioned across a CUDA thread block.
- */
-
-#pragma once
-
-#include "../util_type.cuh"
-#include "../util_ptx.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-template <
-    typename    T,
-    int         BLOCK_DIM_X,
-    int         BLOCK_DIM_Y     = 1,
-    int         BLOCK_DIM_Z     = 1,
-    int         PTX_ARCH        = CUB_PTX_ARCH>
-class BlockAdjacentDifference
-{
-private:
-
-    /******************************************************************************
-     * Constants and type definitions
-     ******************************************************************************/
-
-    /// Constants
-    enum
-    {
-        /// The thread block size in threads
-        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-    };
-
-
-    /// Shared memory storage layout type (last element from each thread's input)
-    struct _TempStorage
-    {
-        T first_items[BLOCK_THREADS];
-        T last_items[BLOCK_THREADS];
-    };
-
-
-    /******************************************************************************
-     * Utility methods
-     ******************************************************************************/
-
-    /// Internal storage allocator
-    __device__ __forceinline__ _TempStorage& PrivateStorage()
-    {
-        __shared__ _TempStorage private_storage;
-        return private_storage;
-    }
-
-
-    /// Specialization for when FlagOp has third index param
-    template <typename FlagOp, bool HAS_PARAM = BinaryOpHasIdxParam<T, FlagOp>::HAS_PARAM>
-    struct ApplyOp
-    {
-        // Apply flag operator
-        static __device__ __forceinline__ T FlagT(FlagOp flag_op, const T &a, const T &b, int idx)
-        {
-            return flag_op(b, a, idx);
-        }
-    };
-
-    /// Specialization for when FlagOp does not have a third index param
-    template <typename FlagOp>
-    struct ApplyOp<FlagOp, false>
-    {
-        // Apply flag operator
-        static __device__ __forceinline__ T FlagT(FlagOp flag_op, const T &a, const T &b, int /*idx*/)
-        {
-            return flag_op(b, a);
-        }
-    };
-
-    /// Templated unrolling of item comparison (inductive case)
-    template <int ITERATION, int MAX_ITERATIONS>
-    struct Iterate
-    {
-        // Head flags
-        template <
-            int             ITEMS_PER_THREAD,
-            typename        FlagT,
-            typename        FlagOp>
-        static __device__ __forceinline__ void FlagHeads(
-            int                     linear_tid,
-            FlagT                   (&flags)[ITEMS_PER_THREAD],         ///< [out] Calling thread's discontinuity head_flags
-            T                       (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-            T                       (&preds)[ITEMS_PER_THREAD],         ///< [out] Calling thread's predecessor items
-            FlagOp                  flag_op)                            ///< [in] Binary boolean flag predicate
-        {
-            preds[ITERATION] = input[ITERATION - 1];
-
-            flags[ITERATION] = ApplyOp<FlagOp>::FlagT(
-                flag_op,
-                preds[ITERATION],
-                input[ITERATION],
-                (linear_tid * ITEMS_PER_THREAD) + ITERATION);
-
-            Iterate<ITERATION + 1, MAX_ITERATIONS>::FlagHeads(linear_tid, flags, input, preds, flag_op);
-        }
-
-        // Tail flags
-        template <
-            int             ITEMS_PER_THREAD,
-            typename        FlagT,
-            typename        FlagOp>
-        static __device__ __forceinline__ void FlagTails(
-            int                     linear_tid,
-            FlagT                   (&flags)[ITEMS_PER_THREAD],         ///< [out] Calling thread's discontinuity head_flags
-            T                       (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-            FlagOp                  flag_op)                            ///< [in] Binary boolean flag predicate
-        {
-            flags[ITERATION] = ApplyOp<FlagOp>::FlagT(
-                flag_op,
-                input[ITERATION],
-                input[ITERATION + 1],
-                (linear_tid * ITEMS_PER_THREAD) + ITERATION + 1);
-
-            Iterate<ITERATION + 1, MAX_ITERATIONS>::FlagTails(linear_tid, flags, input, flag_op);
-        }
-
-    };
-
-    /// Templated unrolling of item comparison (termination case)
-    template <int MAX_ITERATIONS>
-    struct Iterate<MAX_ITERATIONS, MAX_ITERATIONS>
-    {
-        // Head flags
-        template <
-            int             ITEMS_PER_THREAD,
-            typename        FlagT,
-            typename        FlagOp>
-        static __device__ __forceinline__ void FlagHeads(
-            int                     /*linear_tid*/,
-            FlagT                   (&/*flags*/)[ITEMS_PER_THREAD],         ///< [out] Calling thread's discontinuity head_flags
-            T                       (&/*input*/)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-            T                       (&/*preds*/)[ITEMS_PER_THREAD],         ///< [out] Calling thread's predecessor items
-            FlagOp                  /*flag_op*/)                            ///< [in] Binary boolean flag predicate
-        {}
-
-        // Tail flags
-        template <
-            int             ITEMS_PER_THREAD,
-            typename        FlagT,
-            typename        FlagOp>
-        static __device__ __forceinline__ void FlagTails(
-            int                     /*linear_tid*/,
-            FlagT                   (&/*flags*/)[ITEMS_PER_THREAD],         ///< [out] Calling thread's discontinuity head_flags
-            T                       (&/*input*/)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-            FlagOp                  /*flag_op*/)                            ///< [in] Binary boolean flag predicate
-        {}
-    };
-
-
-    /******************************************************************************
-     * Thread fields
-     ******************************************************************************/
-
-    /// Shared storage reference
-    _TempStorage &temp_storage;
-
-    /// Linear thread-id
-    unsigned int linear_tid;
-
-
-public:
-
-    /// \smemstorage{BlockDiscontinuity}
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    /******************************************************************//**
-     * \name Collective constructors
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
-     */
-    __device__ __forceinline__ BlockAdjacentDifference()
-    :
-        temp_storage(PrivateStorage()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    /**
-     * \brief Collective constructor using the specified memory allocation as temporary storage.
-     */
-    __device__ __forceinline__ BlockAdjacentDifference(
-        TempStorage &temp_storage)  ///< [in] Reference to memory allocation having layout type TempStorage
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Head flag operations
-     *********************************************************************/
-    //@{
-
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagHeads(
-        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        T               (&preds)[ITEMS_PER_THREAD],         ///< [out] Calling thread's predecessor items
-        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
-    {
-        // Share last item
-        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
-
-        CTA_SYNC();
-
-        if (linear_tid == 0)
-        {
-            // Set flag for first thread-item (preds[0] is undefined)
-            head_flags[0] = 1;
-        }
-        else
-        {
-            preds[0] = temp_storage.last_items[linear_tid - 1];
-            head_flags[0] = ApplyOp<FlagOp>::FlagT(flag_op, preds[0], input[0], linear_tid * ITEMS_PER_THREAD);
-        }
-
-        // Set head_flags for remaining items
-        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
-    }
-
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagHeads(
-        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        T               (&preds)[ITEMS_PER_THREAD],         ///< [out] Calling thread's predecessor items
-        FlagOp          flag_op,                            ///< [in] Binary boolean flag predicate
-        T               tile_predecessor_item)              ///< [in] <b>[<em>thread</em><sub>0</sub> only]</b> Item with which to compare the first tile item (<tt>input<sub>0</sub></tt> from <em>thread</em><sub>0</sub>).
-    {
-        // Share last item
-        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
-
-        CTA_SYNC();
-
-        // Set flag for first thread-item
-        preds[0] = (linear_tid == 0) ?
-            tile_predecessor_item :              // First thread
-            temp_storage.last_items[linear_tid - 1];
-
-        head_flags[0] = ApplyOp<FlagOp>::FlagT(flag_op, preds[0], input[0], linear_tid * ITEMS_PER_THREAD);
-
-        // Set head_flags for remaining items
-        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
-    }
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagHeads(
-        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
-    {
-        T preds[ITEMS_PER_THREAD];
-        FlagHeads(head_flags, input, preds, flag_op);
-    }
-
-
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagHeads(
-        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        FlagOp          flag_op,                            ///< [in] Binary boolean flag predicate
-        T               tile_predecessor_item)              ///< [in] <b>[<em>thread</em><sub>0</sub> only]</b> Item with which to compare the first tile item (<tt>input<sub>0</sub></tt> from <em>thread</em><sub>0</sub>).
-    {
-        T preds[ITEMS_PER_THREAD];
-        FlagHeads(head_flags, input, preds, flag_op, tile_predecessor_item);
-    }
-
-
-
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagTails(
-        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
-    {
-        // Share first item
-        temp_storage.first_items[linear_tid] = input[0];
-
-        CTA_SYNC();
-
-        // Set flag for last thread-item
-        tail_flags[ITEMS_PER_THREAD - 1] = (linear_tid == BLOCK_THREADS - 1) ?
-            1 :                             // Last thread
-            ApplyOp<FlagOp>::FlagT(
-                flag_op,
-                input[ITEMS_PER_THREAD - 1],
-                temp_storage.first_items[linear_tid + 1],
-                (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
-
-        // Set tail_flags for remaining items
-        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
-    }
-
-
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagTails(
-        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        FlagOp          flag_op,                            ///< [in] Binary boolean flag predicate
-        T               tile_successor_item)                ///< [in] <b>[<em>thread</em><sub><tt>BLOCK_THREADS</tt>-1</sub> only]</b> Item with which to compare the last tile item (<tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> from <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>).
-    {
-        // Share first item
-        temp_storage.first_items[linear_tid] = input[0];
-
-        CTA_SYNC();
-
-        // Set flag for last thread-item
-        T successor_item = (linear_tid == BLOCK_THREADS - 1) ?
-            tile_successor_item :              // Last thread
-            temp_storage.first_items[linear_tid + 1];
-
-        tail_flags[ITEMS_PER_THREAD - 1] = ApplyOp<FlagOp>::FlagT(
-            flag_op,
-            input[ITEMS_PER_THREAD - 1],
-            successor_item,
-            (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
-
-        // Set tail_flags for remaining items
-        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
-    }
-
-
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagHeadsAndTails(
-        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
-        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
-    {
-        // Share first and last items
-        temp_storage.first_items[linear_tid] = input[0];
-        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
-
-        CTA_SYNC();
-
-        T preds[ITEMS_PER_THREAD];
-
-        // Set flag for first thread-item
-        preds[0] = temp_storage.last_items[linear_tid - 1];
-        if (linear_tid == 0)
-        {
-            head_flags[0] = 1;
-        }
-        else
-        {
-            head_flags[0] = ApplyOp<FlagOp>::FlagT(
-                flag_op,
-                preds[0],
-                input[0],
-                linear_tid * ITEMS_PER_THREAD);
-        }
-
-
-        // Set flag for last thread-item
-        tail_flags[ITEMS_PER_THREAD - 1] = (linear_tid == BLOCK_THREADS - 1) ?
-            1 :                             // Last thread
-            ApplyOp<FlagOp>::FlagT(
-                flag_op,
-                input[ITEMS_PER_THREAD - 1],
-                temp_storage.first_items[linear_tid + 1],
-                (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
-
-        // Set head_flags for remaining items
-        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
-
-        // Set tail_flags for remaining items
-        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
-    }
-
-
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagHeadsAndTails(
-        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
-        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
-        T               tile_successor_item,                ///< [in] <b>[<em>thread</em><sub><tt>BLOCK_THREADS</tt>-1</sub> only]</b> Item with which to compare the last tile item (<tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> from <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>).
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
-    {
-        // Share first and last items
-        temp_storage.first_items[linear_tid] = input[0];
-        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
-
-        CTA_SYNC();
-
-        T preds[ITEMS_PER_THREAD];
-
-        // Set flag for first thread-item
-        if (linear_tid == 0)
-        {
-            head_flags[0] = 1;
-        }
-        else
-        {
-            preds[0] = temp_storage.last_items[linear_tid - 1];
-            head_flags[0] = ApplyOp<FlagOp>::FlagT(
-                flag_op,
-                preds[0],
-                input[0],
-                linear_tid * ITEMS_PER_THREAD);
-        }
-
-        // Set flag for last thread-item
-        T successor_item = (linear_tid == BLOCK_THREADS - 1) ?
-            tile_successor_item :              // Last thread
-            temp_storage.first_items[linear_tid + 1];
-
-        tail_flags[ITEMS_PER_THREAD - 1] = ApplyOp<FlagOp>::FlagT(
-            flag_op,
-            input[ITEMS_PER_THREAD - 1],
-            successor_item,
-            (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
-
-        // Set head_flags for remaining items
-        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
-
-        // Set tail_flags for remaining items
-        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
-    }
-
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagHeadsAndTails(
-        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
-        T               tile_predecessor_item,              ///< [in] <b>[<em>thread</em><sub>0</sub> only]</b> Item with which to compare the first tile item (<tt>input<sub>0</sub></tt> from <em>thread</em><sub>0</sub>).
-        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
-    {
-        // Share first and last items
-        temp_storage.first_items[linear_tid] = input[0];
-        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
-
-        CTA_SYNC();
-
-        T preds[ITEMS_PER_THREAD];
-
-        // Set flag for first thread-item
-        preds[0] = (linear_tid == 0) ?
-            tile_predecessor_item :              // First thread
-            temp_storage.last_items[linear_tid - 1];
-
-        head_flags[0] = ApplyOp<FlagOp>::FlagT(
-            flag_op,
-            preds[0],
-            input[0],
-            linear_tid * ITEMS_PER_THREAD);
-
-        // Set flag for last thread-item
-        tail_flags[ITEMS_PER_THREAD - 1] = (linear_tid == BLOCK_THREADS - 1) ?
-            1 :                             // Last thread
-            ApplyOp<FlagOp>::FlagT(
-                flag_op,
-                input[ITEMS_PER_THREAD - 1],
-                temp_storage.first_items[linear_tid + 1],
-                (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
-
-        // Set head_flags for remaining items
-        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
-
-        // Set tail_flags for remaining items
-        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
-    }
-
-
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagHeadsAndTails(
-        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
-        T               tile_predecessor_item,              ///< [in] <b>[<em>thread</em><sub>0</sub> only]</b> Item with which to compare the first tile item (<tt>input<sub>0</sub></tt> from <em>thread</em><sub>0</sub>).
-        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
-        T               tile_successor_item,                ///< [in] <b>[<em>thread</em><sub><tt>BLOCK_THREADS</tt>-1</sub> only]</b> Item with which to compare the last tile item (<tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> from <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>).
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
-    {
-        // Share first and last items
-        temp_storage.first_items[linear_tid] = input[0];
-        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
-
-        CTA_SYNC();
-
-        T preds[ITEMS_PER_THREAD];
-
-        // Set flag for first thread-item
-        preds[0] = (linear_tid == 0) ?
-            tile_predecessor_item :              // First thread
-            temp_storage.last_items[linear_tid - 1];
-
-        head_flags[0] = ApplyOp<FlagOp>::FlagT(
-            flag_op,
-            preds[0],
-            input[0],
-            linear_tid * ITEMS_PER_THREAD);
-
-        // Set flag for last thread-item
-        T successor_item = (linear_tid == BLOCK_THREADS - 1) ?
-            tile_successor_item :              // Last thread
-            temp_storage.first_items[linear_tid + 1];
-
-        tail_flags[ITEMS_PER_THREAD - 1] = ApplyOp<FlagOp>::FlagT(
-            flag_op,
-            input[ITEMS_PER_THREAD - 1],
-            successor_item,
-            (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
-
-        // Set head_flags for remaining items
-        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
-
-        // Set tail_flags for remaining items
-        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
-    }
-
-
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/block/block_discontinuity.cuh b/pfsimulator/third_party/cub/block/block_discontinuity.cuh
deleted file mode 100644
index 503e3e0b0..000000000
--- a/pfsimulator/third_party/cub/block/block_discontinuity.cuh
+++ /dev/null
@@ -1,1148 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * The cub::BlockDiscontinuity class provides [<em>collective</em>](index.html#sec0) methods for flagging discontinuities within an ordered set of items partitioned across a CUDA thread block.
- */
-
-#pragma once
-
-#include "../util_type.cuh"
-#include "../util_ptx.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \brief The BlockDiscontinuity class provides [<em>collective</em>](index.html#sec0) methods for flagging discontinuities within an ordered set of items partitioned across a CUDA thread block. ![](discont_logo.png)
- * \ingroup BlockModule
- *
- * \tparam T                The data type to be flagged.
- * \tparam BLOCK_DIM_X      The thread block length in threads along the X dimension
- * \tparam BLOCK_DIM_Y      <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
- * \tparam BLOCK_DIM_Z      <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
- * \tparam PTX_ARCH         <b>[optional]</b> \ptxversion
- *
- * \par Overview
- * - A set of "head flags" (or "tail flags") is often used to indicate corresponding items
- *   that differ from their predecessors (or successors).  For example, head flags are convenient
- *   for demarcating disjoint data segments as part of a segmented scan or reduction.
- * - \blocked
- *
- * \par Performance Considerations
- * - \granularity
- *
- * \par A Simple Example
- * \blockcollective{BlockDiscontinuity}
- * \par
- * The code snippet below illustrates the head flagging of 512 integer items that
- * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
- * where each thread owns 4 consecutive items.
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/block/block_discontinuity.cuh>
- *
- * __global__ void ExampleKernel(...)
- * {
- *     // Specialize BlockDiscontinuity for a 1D block of 128 threads on type int
- *     typedef cub::BlockDiscontinuity<int, 128> BlockDiscontinuity;
- *
- *     // Allocate shared memory for BlockDiscontinuity
- *     __shared__ typename BlockDiscontinuity::TempStorage temp_storage;
- *
- *     // Obtain a segment of consecutive items that are blocked across threads
- *     int thread_data[4];
- *     ...
- *
- *     // Collectively compute head flags for discontinuities in the segment
- *     int head_flags[4];
- *     BlockDiscontinuity(temp_storage).FlagHeads(head_flags, thread_data, cub::Inequality());
- *
- * \endcode
- * \par
- * Suppose the set of input \p thread_data across the block of threads is
- * <tt>{ [0,0,1,1], [1,1,1,1], [2,3,3,3], [3,4,4,4], ... }</tt>.
- * The corresponding output \p head_flags in those threads will be
- * <tt>{ [1,0,1,0], [0,0,0,0], [1,1,0,0], [0,1,0,0], ... }</tt>.
- *
- * \par Performance Considerations
- * - Incurs zero bank conflicts for most types
- *
- */
-template <
-    typename    T,
-    int         BLOCK_DIM_X,
-    int         BLOCK_DIM_Y     = 1,
-    int         BLOCK_DIM_Z     = 1,
-    int         PTX_ARCH        = CUB_PTX_ARCH>
-class BlockDiscontinuity
-{
-private:
-
-    /******************************************************************************
-     * Constants and type definitions
-     ******************************************************************************/
-
-    /// Constants
-    enum
-    {
-        /// The thread block size in threads
-        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-    };
-
-
-    /// Shared memory storage layout type (last element from each thread's input)
-    struct _TempStorage
-    {
-        T first_items[BLOCK_THREADS];
-        T last_items[BLOCK_THREADS];
-    };
-
-
-    /******************************************************************************
-     * Utility methods
-     ******************************************************************************/
-
-    /// Internal storage allocator
-    __device__ __forceinline__ _TempStorage& PrivateStorage()
-    {
-        __shared__ _TempStorage private_storage;
-        return private_storage;
-    }
-
-
-    /// Specialization for when FlagOp has third index param
-    template <typename FlagOp, bool HAS_PARAM = BinaryOpHasIdxParam<T, FlagOp>::HAS_PARAM>
-    struct ApplyOp
-    {
-        // Apply flag operator
-        static __device__ __forceinline__ bool FlagT(FlagOp flag_op, const T &a, const T &b, int idx)
-        {
-            return flag_op(a, b, idx);
-        }
-    };
-
-    /// Specialization for when FlagOp does not have a third index param
-    template <typename FlagOp>
-    struct ApplyOp<FlagOp, false>
-    {
-        // Apply flag operator
-        static __device__ __forceinline__ bool FlagT(FlagOp flag_op, const T &a, const T &b, int /*idx*/)
-        {
-            return flag_op(a, b);
-        }
-    };
-
-    /// Templated unrolling of item comparison (inductive case)
-    template <int ITERATION, int MAX_ITERATIONS>
-    struct Iterate
-    {
-        // Head flags
-        template <
-            int             ITEMS_PER_THREAD,
-            typename        FlagT,
-            typename        FlagOp>
-        static __device__ __forceinline__ void FlagHeads(
-            int                     linear_tid,
-            FlagT                   (&flags)[ITEMS_PER_THREAD],         ///< [out] Calling thread's discontinuity head_flags
-            T                       (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-            T                       (&preds)[ITEMS_PER_THREAD],         ///< [out] Calling thread's predecessor items
-            FlagOp                  flag_op)                            ///< [in] Binary boolean flag predicate
-        {
-            preds[ITERATION] = input[ITERATION - 1];
-
-            flags[ITERATION] = ApplyOp<FlagOp>::FlagT(
-                flag_op,
-                preds[ITERATION],
-                input[ITERATION],
-                (linear_tid * ITEMS_PER_THREAD) + ITERATION);
-
-            Iterate<ITERATION + 1, MAX_ITERATIONS>::FlagHeads(linear_tid, flags, input, preds, flag_op);
-        }
-
-        // Tail flags
-        template <
-            int             ITEMS_PER_THREAD,
-            typename        FlagT,
-            typename        FlagOp>
-        static __device__ __forceinline__ void FlagTails(
-            int                     linear_tid,
-            FlagT                   (&flags)[ITEMS_PER_THREAD],         ///< [out] Calling thread's discontinuity head_flags
-            T                       (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-            FlagOp                  flag_op)                            ///< [in] Binary boolean flag predicate
-        {
-            flags[ITERATION] = ApplyOp<FlagOp>::FlagT(
-                flag_op,
-                input[ITERATION],
-                input[ITERATION + 1],
-                (linear_tid * ITEMS_PER_THREAD) + ITERATION + 1);
-
-            Iterate<ITERATION + 1, MAX_ITERATIONS>::FlagTails(linear_tid, flags, input, flag_op);
-        }
-
-    };
-
-    /// Templated unrolling of item comparison (termination case)
-    template <int MAX_ITERATIONS>
-    struct Iterate<MAX_ITERATIONS, MAX_ITERATIONS>
-    {
-        // Head flags
-        template <
-            int             ITEMS_PER_THREAD,
-            typename        FlagT,
-            typename        FlagOp>
-        static __device__ __forceinline__ void FlagHeads(
-            int                     /*linear_tid*/,
-            FlagT                   (&/*flags*/)[ITEMS_PER_THREAD],         ///< [out] Calling thread's discontinuity head_flags
-            T                       (&/*input*/)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-            T                       (&/*preds*/)[ITEMS_PER_THREAD],         ///< [out] Calling thread's predecessor items
-            FlagOp                  /*flag_op*/)                            ///< [in] Binary boolean flag predicate
-        {}
-
-        // Tail flags
-        template <
-            int             ITEMS_PER_THREAD,
-            typename        FlagT,
-            typename        FlagOp>
-        static __device__ __forceinline__ void FlagTails(
-            int                     /*linear_tid*/,
-            FlagT                   (&/*flags*/)[ITEMS_PER_THREAD],         ///< [out] Calling thread's discontinuity head_flags
-            T                       (&/*input*/)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-            FlagOp                  /*flag_op*/)                            ///< [in] Binary boolean flag predicate
-        {}
-    };
-
-
-    /******************************************************************************
-     * Thread fields
-     ******************************************************************************/
-
-    /// Shared storage reference
-    _TempStorage &temp_storage;
-
-    /// Linear thread-id
-    unsigned int linear_tid;
-
-
-public:
-
-    /// \smemstorage{BlockDiscontinuity}
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    /******************************************************************//**
-     * \name Collective constructors
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
-     */
-    __device__ __forceinline__ BlockDiscontinuity()
-    :
-        temp_storage(PrivateStorage()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    /**
-     * \brief Collective constructor using the specified memory allocation as temporary storage.
-     */
-    __device__ __forceinline__ BlockDiscontinuity(
-        TempStorage &temp_storage)  ///< [in] Reference to memory allocation having layout type TempStorage
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Head flag operations
-     *********************************************************************/
-    //@{
-
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagHeads(
-        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        T               (&preds)[ITEMS_PER_THREAD],         ///< [out] Calling thread's predecessor items
-        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
-    {
-        // Share last item
-        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
-
-        CTA_SYNC();
-
-        if (linear_tid == 0)
-        {
-            // Set flag for first thread-item (preds[0] is undefined)
-            head_flags[0] = 1;
-        }
-        else
-        {
-            preds[0] = temp_storage.last_items[linear_tid - 1];
-            head_flags[0] = ApplyOp<FlagOp>::FlagT(flag_op, preds[0], input[0], linear_tid * ITEMS_PER_THREAD);
-        }
-
-        // Set head_flags for remaining items
-        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
-    }
-
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagHeads(
-        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        T               (&preds)[ITEMS_PER_THREAD],         ///< [out] Calling thread's predecessor items
-        FlagOp          flag_op,                            ///< [in] Binary boolean flag predicate
-        T               tile_predecessor_item)              ///< [in] <b>[<em>thread</em><sub>0</sub> only]</b> Item with which to compare the first tile item (<tt>input<sub>0</sub></tt> from <em>thread</em><sub>0</sub>).
-    {
-        // Share last item
-        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
-
-        CTA_SYNC();
-
-        // Set flag for first thread-item
-        preds[0] = (linear_tid == 0) ?
-            tile_predecessor_item :              // First thread
-            temp_storage.last_items[linear_tid - 1];
-
-        head_flags[0] = ApplyOp<FlagOp>::FlagT(flag_op, preds[0], input[0], linear_tid * ITEMS_PER_THREAD);
-
-        // Set head_flags for remaining items
-        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
-    }
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-    /**
-     * \brief Sets head flags indicating discontinuities between items partitioned across the thread block, for which the first item has no reference and is always flagged.
-     *
-     * \par
-     * - The flag <tt>head_flags<sub><em>i</em></sub></tt> is set for item
-     *   <tt>input<sub><em>i</em></sub></tt> when
-     *   <tt>flag_op(</tt><em>previous-item</em><tt>, input<sub><em>i</em></sub>)</tt>
-     *   returns \p true (where <em>previous-item</em> is either the preceding item
-     *   in the same thread or the last item in the previous thread).
-     * - For <em>thread</em><sub>0</sub>, item <tt>input<sub>0</sub></tt> is always flagged.
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates the head-flagging of 512 integer items that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_discontinuity.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockDiscontinuity for a 1D block of 128 threads on type int
-     *     typedef cub::BlockDiscontinuity<int, 128> BlockDiscontinuity;
-     *
-     *     // Allocate shared memory for BlockDiscontinuity
-     *     __shared__ typename BlockDiscontinuity::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Collectively compute head flags for discontinuities in the segment
-     *     int head_flags[4];
-     *     BlockDiscontinuity(temp_storage).FlagHeads(head_flags, thread_data, cub::Inequality());
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is
-     * <tt>{ [0,0,1,1], [1,1,1,1], [2,3,3,3], [3,4,4,4], ... }</tt>.
-     * The corresponding output \p head_flags in those threads will be
-     * <tt>{ [1,0,1,0], [0,0,0,0], [1,1,0,0], [0,1,0,0], ... }</tt>.
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam FlagT                <b>[inferred]</b> The flag type (must be an integer type)
-     * \tparam FlagOp               <b>[inferred]</b> Binary predicate functor type having member <tt>T operator()(const T &a, const T &b)</tt> or member <tt>T operator()(const T &a, const T &b, unsigned int b_index)</tt>, and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false.  \p b_index is the rank of b in the aggregate tile of data.
-     */
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagHeads(
-        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
-    {
-        T preds[ITEMS_PER_THREAD];
-        FlagHeads(head_flags, input, preds, flag_op);
-    }
-
-
-    /**
-     * \brief Sets head flags indicating discontinuities between items partitioned across the thread block.
-     *
-     * \par
-     * - The flag <tt>head_flags<sub><em>i</em></sub></tt> is set for item
-     *   <tt>input<sub><em>i</em></sub></tt> when
-     *   <tt>flag_op(</tt><em>previous-item</em><tt>, input<sub><em>i</em></sub>)</tt>
-     *   returns \p true (where <em>previous-item</em> is either the preceding item
-     *   in the same thread or the last item in the previous thread).
-     * - For <em>thread</em><sub>0</sub>, item <tt>input<sub>0</sub></tt> is compared
-     *   against \p tile_predecessor_item.
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates the head-flagging of 512 integer items that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_discontinuity.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockDiscontinuity for a 1D block of 128 threads on type int
-     *     typedef cub::BlockDiscontinuity<int, 128> BlockDiscontinuity;
-     *
-     *     // Allocate shared memory for BlockDiscontinuity
-     *     __shared__ typename BlockDiscontinuity::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Have thread0 obtain the predecessor item for the entire tile
-     *     int tile_predecessor_item;
-     *     if (threadIdx.x == 0) tile_predecessor_item == ...
-     *
-     *     // Collectively compute head flags for discontinuities in the segment
-     *     int head_flags[4];
-     *     BlockDiscontinuity(temp_storage).FlagHeads(
-     *         head_flags, thread_data, cub::Inequality(), tile_predecessor_item);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is
-     * <tt>{ [0,0,1,1], [1,1,1,1], [2,3,3,3], [3,4,4,4], ... }</tt>,
-     * and that \p tile_predecessor_item is \p 0.  The corresponding output \p head_flags in those threads will be
-     * <tt>{ [0,0,1,0], [0,0,0,0], [1,1,0,0], [0,1,0,0], ... }</tt>.
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam FlagT                <b>[inferred]</b> The flag type (must be an integer type)
-     * \tparam FlagOp               <b>[inferred]</b> Binary predicate functor type having member <tt>T operator()(const T &a, const T &b)</tt> or member <tt>T operator()(const T &a, const T &b, unsigned int b_index)</tt>, and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false.  \p b_index is the rank of b in the aggregate tile of data.
-     */
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagHeads(
-        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        FlagOp          flag_op,                            ///< [in] Binary boolean flag predicate
-        T               tile_predecessor_item)              ///< [in] <b>[<em>thread</em><sub>0</sub> only]</b> Item with which to compare the first tile item (<tt>input<sub>0</sub></tt> from <em>thread</em><sub>0</sub>).
-    {
-        T preds[ITEMS_PER_THREAD];
-        FlagHeads(head_flags, input, preds, flag_op, tile_predecessor_item);
-    }
-
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Tail flag operations
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Sets tail flags indicating discontinuities between items partitioned across the thread block, for which the last item has no reference and is always flagged.
-     *
-     * \par
-     * - The flag <tt>tail_flags<sub><em>i</em></sub></tt> is set for item
-     *   <tt>input<sub><em>i</em></sub></tt> when
-     *   <tt>flag_op(input<sub><em>i</em></sub>, </tt><em>next-item</em><tt>)</tt>
-     *   returns \p true (where <em>next-item</em> is either the next item
-     *   in the same thread or the first item in the next thread).
-     * - For <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>, item
-     *   <tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> is always flagged.
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates the tail-flagging of 512 integer items that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_discontinuity.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockDiscontinuity for a 1D block of 128 threads on type int
-     *     typedef cub::BlockDiscontinuity<int, 128> BlockDiscontinuity;
-     *
-     *     // Allocate shared memory for BlockDiscontinuity
-     *     __shared__ typename BlockDiscontinuity::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Collectively compute tail flags for discontinuities in the segment
-     *     int tail_flags[4];
-     *     BlockDiscontinuity(temp_storage).FlagTails(tail_flags, thread_data, cub::Inequality());
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is
-     * <tt>{ [0,0,1,1], [1,1,1,1], [2,3,3,3], ..., [124,125,125,125] }</tt>.
-     * The corresponding output \p tail_flags in those threads will be
-     * <tt>{ [0,1,0,0], [0,0,0,1], [1,0,0,...], ..., [1,0,0,1] }</tt>.
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam FlagT                <b>[inferred]</b> The flag type (must be an integer type)
-     * \tparam FlagOp               <b>[inferred]</b> Binary predicate functor type having member <tt>T operator()(const T &a, const T &b)</tt> or member <tt>T operator()(const T &a, const T &b, unsigned int b_index)</tt>, and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false.  \p b_index is the rank of b in the aggregate tile of data.
-     */
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagTails(
-        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
-    {
-        // Share first item
-        temp_storage.first_items[linear_tid] = input[0];
-
-        CTA_SYNC();
-
-        // Set flag for last thread-item
-        tail_flags[ITEMS_PER_THREAD - 1] = (linear_tid == BLOCK_THREADS - 1) ?
-            1 :                             // Last thread
-            ApplyOp<FlagOp>::FlagT(
-                flag_op,
-                input[ITEMS_PER_THREAD - 1],
-                temp_storage.first_items[linear_tid + 1],
-                (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
-
-        // Set tail_flags for remaining items
-        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
-    }
-
-
-    /**
-     * \brief Sets tail flags indicating discontinuities between items partitioned across the thread block.
-     *
-     * \par
-     * - The flag <tt>tail_flags<sub><em>i</em></sub></tt> is set for item
-     *   <tt>input<sub><em>i</em></sub></tt> when
-     *   <tt>flag_op(input<sub><em>i</em></sub>, </tt><em>next-item</em><tt>)</tt>
-     *   returns \p true (where <em>next-item</em> is either the next item
-     *   in the same thread or the first item in the next thread).
-     * - For <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>, item
-     *   <tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> is compared
-     *   against \p tile_successor_item.
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates the tail-flagging of 512 integer items that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_discontinuity.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockDiscontinuity for a 1D block of 128 threads on type int
-     *     typedef cub::BlockDiscontinuity<int, 128> BlockDiscontinuity;
-     *
-     *     // Allocate shared memory for BlockDiscontinuity
-     *     __shared__ typename BlockDiscontinuity::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Have thread127 obtain the successor item for the entire tile
-     *     int tile_successor_item;
-     *     if (threadIdx.x == 127) tile_successor_item == ...
-     *
-     *     // Collectively compute tail flags for discontinuities in the segment
-     *     int tail_flags[4];
-     *     BlockDiscontinuity(temp_storage).FlagTails(
-     *         tail_flags, thread_data, cub::Inequality(), tile_successor_item);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is
-     * <tt>{ [0,0,1,1], [1,1,1,1], [2,3,3,3], ..., [124,125,125,125] }</tt>
-     * and that \p tile_successor_item is \p 125.  The corresponding output \p tail_flags in those threads will be
-     * <tt>{ [0,1,0,0], [0,0,0,1], [1,0,0,...], ..., [1,0,0,0] }</tt>.
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam FlagT                <b>[inferred]</b> The flag type (must be an integer type)
-     * \tparam FlagOp               <b>[inferred]</b> Binary predicate functor type having member <tt>T operator()(const T &a, const T &b)</tt> or member <tt>T operator()(const T &a, const T &b, unsigned int b_index)</tt>, and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false.  \p b_index is the rank of b in the aggregate tile of data.
-     */
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagTails(
-        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        FlagOp          flag_op,                            ///< [in] Binary boolean flag predicate
-        T               tile_successor_item)                ///< [in] <b>[<em>thread</em><sub><tt>BLOCK_THREADS</tt>-1</sub> only]</b> Item with which to compare the last tile item (<tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> from <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>).
-    {
-        // Share first item
-        temp_storage.first_items[linear_tid] = input[0];
-
-        CTA_SYNC();
-
-        // Set flag for last thread-item
-        T successor_item = (linear_tid == BLOCK_THREADS - 1) ?
-            tile_successor_item :              // Last thread
-            temp_storage.first_items[linear_tid + 1];
-
-        tail_flags[ITEMS_PER_THREAD - 1] = ApplyOp<FlagOp>::FlagT(
-            flag_op,
-            input[ITEMS_PER_THREAD - 1],
-            successor_item,
-            (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
-
-        // Set tail_flags for remaining items
-        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
-    }
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Head & tail flag operations
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Sets both head and tail flags indicating discontinuities between items partitioned across the thread block.
-     *
-     * \par
-     * - The flag <tt>head_flags<sub><em>i</em></sub></tt> is set for item
-     *   <tt>input<sub><em>i</em></sub></tt> when
-     *   <tt>flag_op(</tt><em>previous-item</em><tt>, input<sub><em>i</em></sub>)</tt>
-     *   returns \p true (where <em>previous-item</em> is either the preceding item
-     *   in the same thread or the last item in the previous thread).
-     * - For <em>thread</em><sub>0</sub>, item <tt>input<sub>0</sub></tt> is always flagged.
-     * - The flag <tt>tail_flags<sub><em>i</em></sub></tt> is set for item
-     *   <tt>input<sub><em>i</em></sub></tt> when
-     *   <tt>flag_op(input<sub><em>i</em></sub>, </tt><em>next-item</em><tt>)</tt>
-     *   returns \p true (where <em>next-item</em> is either the next item
-     *   in the same thread or the first item in the next thread).
-     * - For <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>, item
-     *   <tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> is always flagged.
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates the head- and tail-flagging of 512 integer items that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_discontinuity.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockDiscontinuity for a 1D block of 128 threads on type int
-     *     typedef cub::BlockDiscontinuity<int, 128> BlockDiscontinuity;
-     *
-     *     // Allocate shared memory for BlockDiscontinuity
-     *     __shared__ typename BlockDiscontinuity::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Collectively compute head and flags for discontinuities in the segment
-     *     int head_flags[4];
-     *     int tail_flags[4];
-     *     BlockDiscontinuity(temp_storage).FlagTails(
-     *         head_flags, tail_flags, thread_data, cub::Inequality());
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is
-     * <tt>{ [0,0,1,1], [1,1,1,1], [2,3,3,3], ..., [124,125,125,125] }</tt>
-     * and that the tile_successor_item is \p 125.  The corresponding output \p head_flags
-     * in those threads will be <tt>{ [1,0,1,0], [0,0,0,0], [1,1,0,0], [0,1,0,0], ... }</tt>.
-     * and the corresponding output \p tail_flags in those threads will be
-     * <tt>{ [0,1,0,0], [0,0,0,1], [1,0,0,...], ..., [1,0,0,1] }</tt>.
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam FlagT                <b>[inferred]</b> The flag type (must be an integer type)
-     * \tparam FlagOp               <b>[inferred]</b> Binary predicate functor type having member <tt>T operator()(const T &a, const T &b)</tt> or member <tt>T operator()(const T &a, const T &b, unsigned int b_index)</tt>, and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false.  \p b_index is the rank of b in the aggregate tile of data.
-     */
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagHeadsAndTails(
-        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
-        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
-    {
-        // Share first and last items
-        temp_storage.first_items[linear_tid] = input[0];
-        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
-
-        CTA_SYNC();
-
-        T preds[ITEMS_PER_THREAD];
-
-        // Set flag for first thread-item
-        preds[0] = temp_storage.last_items[linear_tid - 1];
-        if (linear_tid == 0)
-        {
-            head_flags[0] = 1;
-        }
-        else
-        {
-            head_flags[0] = ApplyOp<FlagOp>::FlagT(
-                flag_op,
-                preds[0],
-                input[0],
-                linear_tid * ITEMS_PER_THREAD);
-        }
-
-
-        // Set flag for last thread-item
-        tail_flags[ITEMS_PER_THREAD - 1] = (linear_tid == BLOCK_THREADS - 1) ?
-            1 :                             // Last thread
-            ApplyOp<FlagOp>::FlagT(
-                flag_op,
-                input[ITEMS_PER_THREAD - 1],
-                temp_storage.first_items[linear_tid + 1],
-                (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
-
-        // Set head_flags for remaining items
-        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
-
-        // Set tail_flags for remaining items
-        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
-    }
-
-
-    /**
-     * \brief Sets both head and tail flags indicating discontinuities between items partitioned across the thread block.
-     *
-     * \par
-     * - The flag <tt>head_flags<sub><em>i</em></sub></tt> is set for item
-     *   <tt>input<sub><em>i</em></sub></tt> when
-     *   <tt>flag_op(</tt><em>previous-item</em><tt>, input<sub><em>i</em></sub>)</tt>
-     *   returns \p true (where <em>previous-item</em> is either the preceding item
-     *   in the same thread or the last item in the previous thread).
-     * - For <em>thread</em><sub>0</sub>, item <tt>input<sub>0</sub></tt> is always flagged.
-     * - The flag <tt>tail_flags<sub><em>i</em></sub></tt> is set for item
-     *   <tt>input<sub><em>i</em></sub></tt> when
-     *   <tt>flag_op(input<sub><em>i</em></sub>, </tt><em>next-item</em><tt>)</tt>
-     *   returns \p true (where <em>next-item</em> is either the next item
-     *   in the same thread or the first item in the next thread).
-     * - For <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>, item
-     *   <tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> is compared
-     *   against \p tile_predecessor_item.
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates the head- and tail-flagging of 512 integer items that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_discontinuity.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockDiscontinuity for a 1D block of 128 threads on type int
-     *     typedef cub::BlockDiscontinuity<int, 128> BlockDiscontinuity;
-     *
-     *     // Allocate shared memory for BlockDiscontinuity
-     *     __shared__ typename BlockDiscontinuity::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Have thread127 obtain the successor item for the entire tile
-     *     int tile_successor_item;
-     *     if (threadIdx.x == 127) tile_successor_item == ...
-     *
-     *     // Collectively compute head and flags for discontinuities in the segment
-     *     int head_flags[4];
-     *     int tail_flags[4];
-     *     BlockDiscontinuity(temp_storage).FlagTails(
-     *         head_flags, tail_flags, tile_successor_item, thread_data, cub::Inequality());
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is
-     * <tt>{ [0,0,1,1], [1,1,1,1], [2,3,3,3], ..., [124,125,125,125] }</tt>
-     * and that the tile_successor_item is \p 125.  The corresponding output \p head_flags
-     * in those threads will be <tt>{ [1,0,1,0], [0,0,0,0], [1,1,0,0], [0,1,0,0], ... }</tt>.
-     * and the corresponding output \p tail_flags in those threads will be
-     * <tt>{ [0,1,0,0], [0,0,0,1], [1,0,0,...], ..., [1,0,0,0] }</tt>.
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam FlagT                <b>[inferred]</b> The flag type (must be an integer type)
-     * \tparam FlagOp               <b>[inferred]</b> Binary predicate functor type having member <tt>T operator()(const T &a, const T &b)</tt> or member <tt>T operator()(const T &a, const T &b, unsigned int b_index)</tt>, and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false.  \p b_index is the rank of b in the aggregate tile of data.
-     */
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagHeadsAndTails(
-        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
-        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
-        T               tile_successor_item,                ///< [in] <b>[<em>thread</em><sub><tt>BLOCK_THREADS</tt>-1</sub> only]</b> Item with which to compare the last tile item (<tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> from <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>).
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
-    {
-        // Share first and last items
-        temp_storage.first_items[linear_tid] = input[0];
-        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
-
-        CTA_SYNC();
-
-        T preds[ITEMS_PER_THREAD];
-
-        // Set flag for first thread-item
-        if (linear_tid == 0)
-        {
-            head_flags[0] = 1;
-        }
-        else
-        {
-            preds[0] = temp_storage.last_items[linear_tid - 1];
-            head_flags[0] = ApplyOp<FlagOp>::FlagT(
-                flag_op,
-                preds[0],
-                input[0],
-                linear_tid * ITEMS_PER_THREAD);
-        }
-
-        // Set flag for last thread-item
-        T successor_item = (linear_tid == BLOCK_THREADS - 1) ?
-            tile_successor_item :              // Last thread
-            temp_storage.first_items[linear_tid + 1];
-
-        tail_flags[ITEMS_PER_THREAD - 1] = ApplyOp<FlagOp>::FlagT(
-            flag_op,
-            input[ITEMS_PER_THREAD - 1],
-            successor_item,
-            (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
-
-        // Set head_flags for remaining items
-        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
-
-        // Set tail_flags for remaining items
-        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
-    }
-
-
-    /**
-     * \brief Sets both head and tail flags indicating discontinuities between items partitioned across the thread block.
-     *
-     * \par
-     * - The flag <tt>head_flags<sub><em>i</em></sub></tt> is set for item
-     *   <tt>input<sub><em>i</em></sub></tt> when
-     *   <tt>flag_op(</tt><em>previous-item</em><tt>, input<sub><em>i</em></sub>)</tt>
-     *   returns \p true (where <em>previous-item</em> is either the preceding item
-     *   in the same thread or the last item in the previous thread).
-     * - For <em>thread</em><sub>0</sub>, item <tt>input<sub>0</sub></tt> is compared
-     *   against \p tile_predecessor_item.
-     * - The flag <tt>tail_flags<sub><em>i</em></sub></tt> is set for item
-     *   <tt>input<sub><em>i</em></sub></tt> when
-     *   <tt>flag_op(input<sub><em>i</em></sub>, </tt><em>next-item</em><tt>)</tt>
-     *   returns \p true (where <em>next-item</em> is either the next item
-     *   in the same thread or the first item in the next thread).
-     * - For <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>, item
-     *   <tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> is always flagged.
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates the head- and tail-flagging of 512 integer items that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_discontinuity.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockDiscontinuity for a 1D block of 128 threads on type int
-     *     typedef cub::BlockDiscontinuity<int, 128> BlockDiscontinuity;
-     *
-     *     // Allocate shared memory for BlockDiscontinuity
-     *     __shared__ typename BlockDiscontinuity::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Have thread0 obtain the predecessor item for the entire tile
-     *     int tile_predecessor_item;
-     *     if (threadIdx.x == 0) tile_predecessor_item == ...
-     *
-     *     // Have thread127 obtain the successor item for the entire tile
-     *     int tile_successor_item;
-     *     if (threadIdx.x == 127) tile_successor_item == ...
-     *
-     *     // Collectively compute head and flags for discontinuities in the segment
-     *     int head_flags[4];
-     *     int tail_flags[4];
-     *     BlockDiscontinuity(temp_storage).FlagTails(
-     *         head_flags, tile_predecessor_item, tail_flags, tile_successor_item,
-     *         thread_data, cub::Inequality());
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is
-     * <tt>{ [0,0,1,1], [1,1,1,1], [2,3,3,3], ..., [124,125,125,125] }</tt>,
-     * that the \p tile_predecessor_item is \p 0, and that the
-     * \p tile_successor_item is \p 125.  The corresponding output \p head_flags
-     * in those threads will be <tt>{ [0,0,1,0], [0,0,0,0], [1,1,0,0], [0,1,0,0], ... }</tt>.
-     * and the corresponding output \p tail_flags in those threads will be
-     * <tt>{ [0,1,0,0], [0,0,0,1], [1,0,0,...], ..., [1,0,0,1] }</tt>.
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam FlagT                <b>[inferred]</b> The flag type (must be an integer type)
-     * \tparam FlagOp               <b>[inferred]</b> Binary predicate functor type having member <tt>T operator()(const T &a, const T &b)</tt> or member <tt>T operator()(const T &a, const T &b, unsigned int b_index)</tt>, and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false.  \p b_index is the rank of b in the aggregate tile of data.
-     */
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagHeadsAndTails(
-        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
-        T               tile_predecessor_item,              ///< [in] <b>[<em>thread</em><sub>0</sub> only]</b> Item with which to compare the first tile item (<tt>input<sub>0</sub></tt> from <em>thread</em><sub>0</sub>).
-        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
-    {
-        // Share first and last items
-        temp_storage.first_items[linear_tid] = input[0];
-        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
-
-        CTA_SYNC();
-
-        T preds[ITEMS_PER_THREAD];
-
-        // Set flag for first thread-item
-        preds[0] = (linear_tid == 0) ?
-            tile_predecessor_item :              // First thread
-            temp_storage.last_items[linear_tid - 1];
-
-        head_flags[0] = ApplyOp<FlagOp>::FlagT(
-            flag_op,
-            preds[0],
-            input[0],
-            linear_tid * ITEMS_PER_THREAD);
-
-        // Set flag for last thread-item
-        tail_flags[ITEMS_PER_THREAD - 1] = (linear_tid == BLOCK_THREADS - 1) ?
-            1 :                             // Last thread
-            ApplyOp<FlagOp>::FlagT(
-                flag_op,
-                input[ITEMS_PER_THREAD - 1],
-                temp_storage.first_items[linear_tid + 1],
-                (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
-
-        // Set head_flags for remaining items
-        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
-
-        // Set tail_flags for remaining items
-        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
-    }
-
-
-    /**
-     * \brief Sets both head and tail flags indicating discontinuities between items partitioned across the thread block.
-     *
-     * \par
-     * - The flag <tt>head_flags<sub><em>i</em></sub></tt> is set for item
-     *   <tt>input<sub><em>i</em></sub></tt> when
-     *   <tt>flag_op(</tt><em>previous-item</em><tt>, input<sub><em>i</em></sub>)</tt>
-     *   returns \p true (where <em>previous-item</em> is either the preceding item
-     *   in the same thread or the last item in the previous thread).
-     * - For <em>thread</em><sub>0</sub>, item <tt>input<sub>0</sub></tt> is compared
-     *   against \p tile_predecessor_item.
-     * - The flag <tt>tail_flags<sub><em>i</em></sub></tt> is set for item
-     *   <tt>input<sub><em>i</em></sub></tt> when
-     *   <tt>flag_op(input<sub><em>i</em></sub>, </tt><em>next-item</em><tt>)</tt>
-     *   returns \p true (where <em>next-item</em> is either the next item
-     *   in the same thread or the first item in the next thread).
-     * - For <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>, item
-     *   <tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> is compared
-     *   against \p tile_successor_item.
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates the head- and tail-flagging of 512 integer items that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_discontinuity.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockDiscontinuity for a 1D block of 128 threads on type int
-     *     typedef cub::BlockDiscontinuity<int, 128> BlockDiscontinuity;
-     *
-     *     // Allocate shared memory for BlockDiscontinuity
-     *     __shared__ typename BlockDiscontinuity::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Have thread0 obtain the predecessor item for the entire tile
-     *     int tile_predecessor_item;
-     *     if (threadIdx.x == 0) tile_predecessor_item == ...
-     *
-     *     // Have thread127 obtain the successor item for the entire tile
-     *     int tile_successor_item;
-     *     if (threadIdx.x == 127) tile_successor_item == ...
-     *
-     *     // Collectively compute head and flags for discontinuities in the segment
-     *     int head_flags[4];
-     *     int tail_flags[4];
-     *     BlockDiscontinuity(temp_storage).FlagTails(
-     *         head_flags, tile_predecessor_item, tail_flags, tile_successor_item,
-     *         thread_data, cub::Inequality());
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is
-     * <tt>{ [0,0,1,1], [1,1,1,1], [2,3,3,3], ..., [124,125,125,125] }</tt>,
-     * that the \p tile_predecessor_item is \p 0, and that the
-     * \p tile_successor_item is \p 125.  The corresponding output \p head_flags
-     * in those threads will be <tt>{ [0,0,1,0], [0,0,0,0], [1,1,0,0], [0,1,0,0], ... }</tt>.
-     * and the corresponding output \p tail_flags in those threads will be
-     * <tt>{ [0,1,0,0], [0,0,0,1], [1,0,0,...], ..., [1,0,0,0] }</tt>.
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam FlagT                <b>[inferred]</b> The flag type (must be an integer type)
-     * \tparam FlagOp               <b>[inferred]</b> Binary predicate functor type having member <tt>T operator()(const T &a, const T &b)</tt> or member <tt>T operator()(const T &a, const T &b, unsigned int b_index)</tt>, and returning \p true if a discontinuity exists between \p a and \p b, otherwise \p false.  \p b_index is the rank of b in the aggregate tile of data.
-     */
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        FlagT,
-        typename        FlagOp>
-    __device__ __forceinline__ void FlagHeadsAndTails(
-        FlagT           (&head_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity head_flags
-        T               tile_predecessor_item,              ///< [in] <b>[<em>thread</em><sub>0</sub> only]</b> Item with which to compare the first tile item (<tt>input<sub>0</sub></tt> from <em>thread</em><sub>0</sub>).
-        FlagT           (&tail_flags)[ITEMS_PER_THREAD],    ///< [out] Calling thread's discontinuity tail_flags
-        T               tile_successor_item,                ///< [in] <b>[<em>thread</em><sub><tt>BLOCK_THREADS</tt>-1</sub> only]</b> Item with which to compare the last tile item (<tt>input</tt><sub><em>ITEMS_PER_THREAD</em>-1</sub> from <em>thread</em><sub><em>BLOCK_THREADS</em>-1</sub>).
-        T               (&input)[ITEMS_PER_THREAD],         ///< [in] Calling thread's input items
-        FlagOp          flag_op)                            ///< [in] Binary boolean flag predicate
-    {
-        // Share first and last items
-        temp_storage.first_items[linear_tid] = input[0];
-        temp_storage.last_items[linear_tid] = input[ITEMS_PER_THREAD - 1];
-
-        CTA_SYNC();
-
-        T preds[ITEMS_PER_THREAD];
-
-        // Set flag for first thread-item
-        preds[0] = (linear_tid == 0) ?
-            tile_predecessor_item :              // First thread
-            temp_storage.last_items[linear_tid - 1];
-
-        head_flags[0] = ApplyOp<FlagOp>::FlagT(
-            flag_op,
-            preds[0],
-            input[0],
-            linear_tid * ITEMS_PER_THREAD);
-
-        // Set flag for last thread-item
-        T successor_item = (linear_tid == BLOCK_THREADS - 1) ?
-            tile_successor_item :              // Last thread
-            temp_storage.first_items[linear_tid + 1];
-
-        tail_flags[ITEMS_PER_THREAD - 1] = ApplyOp<FlagOp>::FlagT(
-            flag_op,
-            input[ITEMS_PER_THREAD - 1],
-            successor_item,
-            (linear_tid * ITEMS_PER_THREAD) + ITEMS_PER_THREAD);
-
-        // Set head_flags for remaining items
-        Iterate<1, ITEMS_PER_THREAD>::FlagHeads(linear_tid, head_flags, input, preds, flag_op);
-
-        // Set tail_flags for remaining items
-        Iterate<0, ITEMS_PER_THREAD - 1>::FlagTails(linear_tid, tail_flags, input, flag_op);
-    }
-
-
-
-
-    //@}  end member group
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/block/block_exchange.cuh b/pfsimulator/third_party/cub/block/block_exchange.cuh
deleted file mode 100644
index 3ae993439..000000000
--- a/pfsimulator/third_party/cub/block/block_exchange.cuh
+++ /dev/null
@@ -1,1248 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * The cub::BlockExchange class provides [<em>collective</em>](index.html#sec0) methods for rearranging data partitioned across a CUDA thread block.
- */
-
-#pragma once
-
-#include "../util_ptx.cuh"
-#include "../util_arch.cuh"
-#include "../util_macro.cuh"
-#include "../util_type.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \brief The BlockExchange class provides [<em>collective</em>](index.html#sec0) methods for rearranging data partitioned across a CUDA thread block. ![](transpose_logo.png)
- * \ingroup BlockModule
- *
- * \tparam T                    The data type to be exchanged.
- * \tparam BLOCK_DIM_X          The thread block length in threads along the X dimension
- * \tparam ITEMS_PER_THREAD     The number of items partitioned onto each thread.
- * \tparam WARP_TIME_SLICING    <b>[optional]</b> When \p true, only use enough shared memory for a single warp's worth of tile data, time-slicing the block-wide exchange over multiple synchronized rounds.  Yields a smaller memory footprint at the expense of decreased parallelism.  (Default: false)
- * \tparam BLOCK_DIM_Y          <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
- * \tparam BLOCK_DIM_Z          <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
- * \tparam PTX_ARCH             <b>[optional]</b> \ptxversion
- *
- * \par Overview
- * - It is commonplace for blocks of threads to rearrange data items between
- *   threads.  For example, the device-accessible memory subsystem prefers access patterns
- *   where data items are "striped" across threads (where consecutive threads access consecutive items),
- *   yet most block-wide operations prefer a "blocked" partitioning of items across threads
- *   (where consecutive items belong to a single thread).
- * - BlockExchange supports the following types of data exchanges:
- *   - Transposing between [<em>blocked</em>](index.html#sec5sec3) and [<em>striped</em>](index.html#sec5sec3) arrangements
- *   - Transposing between [<em>blocked</em>](index.html#sec5sec3) and [<em>warp-striped</em>](index.html#sec5sec3) arrangements
- *   - Scattering ranked items to a [<em>blocked arrangement</em>](index.html#sec5sec3)
- *   - Scattering ranked items to a [<em>striped arrangement</em>](index.html#sec5sec3)
- * - \rowmajor
- *
- * \par A Simple Example
- * \blockcollective{BlockExchange}
- * \par
- * The code snippet below illustrates the conversion from a "blocked" to a "striped" arrangement
- * of 512 integer items partitioned across 128 threads where each thread owns 4 items.
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/block/block_exchange.cuh>
- *
- * __global__ void ExampleKernel(int *d_data, ...)
- * {
- *     // Specialize BlockExchange for a 1D block of 128 threads owning 4 integer items each
- *     typedef cub::BlockExchange<int, 128, 4> BlockExchange;
- *
- *     // Allocate shared memory for BlockExchange
- *     __shared__ typename BlockExchange::TempStorage temp_storage;
- *
- *     // Load a tile of data striped across threads
- *     int thread_data[4];
- *     cub::LoadDirectStriped<128>(threadIdx.x, d_data, thread_data);
- *
- *     // Collectively exchange data into a blocked arrangement across threads
- *     BlockExchange(temp_storage).StripedToBlocked(thread_data);
- *
- * \endcode
- * \par
- * Suppose the set of striped input \p thread_data across the block of threads is
- * <tt>{ [0,128,256,384], [1,129,257,385], ..., [127,255,383,511] }</tt>.
- * The corresponding output \p thread_data in those threads will be
- * <tt>{ [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }</tt>.
- *
- * \par Performance Considerations
- * - Proper device-specific padding ensures zero bank conflicts for most types.
- *
- */
-template <
-    typename    InputT,
-    int         BLOCK_DIM_X,
-    int         ITEMS_PER_THREAD,
-    bool        WARP_TIME_SLICING   = false,
-    int         BLOCK_DIM_Y         = 1,
-    int         BLOCK_DIM_Z         = 1,
-    int         PTX_ARCH            = CUB_PTX_ARCH>
-class BlockExchange
-{
-private:
-
-    /******************************************************************************
-     * Constants
-     ******************************************************************************/
-
-    /// Constants
-    enum
-    {
-        /// The thread block size in threads
-        BLOCK_THREADS               = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-
-        LOG_WARP_THREADS            = CUB_LOG_WARP_THREADS(PTX_ARCH),
-        WARP_THREADS                = 1 << LOG_WARP_THREADS,
-        WARPS                       = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS,
-
-        LOG_SMEM_BANKS              = CUB_LOG_SMEM_BANKS(PTX_ARCH),
-        SMEM_BANKS                  = 1 << LOG_SMEM_BANKS,
-
-        TILE_ITEMS                  = BLOCK_THREADS * ITEMS_PER_THREAD,
-
-        TIME_SLICES                 = (WARP_TIME_SLICING) ? WARPS : 1,
-
-        TIME_SLICED_THREADS         = (WARP_TIME_SLICING) ? CUB_MIN(BLOCK_THREADS, WARP_THREADS) : BLOCK_THREADS,
-        TIME_SLICED_ITEMS           = TIME_SLICED_THREADS * ITEMS_PER_THREAD,
-
-        WARP_TIME_SLICED_THREADS    = CUB_MIN(BLOCK_THREADS, WARP_THREADS),
-        WARP_TIME_SLICED_ITEMS      = WARP_TIME_SLICED_THREADS * ITEMS_PER_THREAD,
-
-        // Insert padding to avoid bank conflicts during raking when items per thread is a power of two and > 4 (otherwise we can typically use 128b loads)
-        INSERT_PADDING              = (ITEMS_PER_THREAD > 4) && (PowerOfTwo<ITEMS_PER_THREAD>::VALUE),
-        PADDING_ITEMS               = (INSERT_PADDING) ? (TIME_SLICED_ITEMS >> LOG_SMEM_BANKS) : 0,
-    };
-
-    /******************************************************************************
-     * Type definitions
-     ******************************************************************************/
-
-    /// Shared memory storage layout type
-    struct __align__(16) _TempStorage
-    {
-        InputT buff[TIME_SLICED_ITEMS + PADDING_ITEMS];
-    };
-
-public:
-
-    /// \smemstorage{BlockExchange}
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-private:
-
-
-    /******************************************************************************
-     * Thread fields
-     ******************************************************************************/
-
-    /// Shared storage reference
-    _TempStorage &temp_storage;
-
-    /// Linear thread-id
-    unsigned int linear_tid;
-    unsigned int lane_id;
-    unsigned int warp_id;
-    unsigned int warp_offset;
-
-
-    /******************************************************************************
-     * Utility methods
-     ******************************************************************************/
-
-    /// Internal storage allocator
-    __device__ __forceinline__ _TempStorage& PrivateStorage()
-    {
-        __shared__ _TempStorage private_storage;
-        return private_storage;
-    }
-
-
-    /**
-     * Transposes data items from <em>blocked</em> arrangement to <em>striped</em> arrangement.  Specialized for no timeslicing.
-     */
-    template <typename OutputT>
-    __device__ __forceinline__ void BlockedToStriped(
-        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        Int2Type<false> /*time_slicing*/)
-    {
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            int item_offset = (linear_tid * ITEMS_PER_THREAD) + ITEM;
-            if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-            temp_storage.buff[item_offset] = input_items[ITEM];
-        }
-
-        CTA_SYNC();
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            int item_offset = int(ITEM * BLOCK_THREADS) + linear_tid;
-            if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-            output_items[ITEM] = temp_storage.buff[item_offset];
-        }
-    }
-
-
-    /**
-     * Transposes data items from <em>blocked</em> arrangement to <em>striped</em> arrangement.  Specialized for warp-timeslicing.
-     */
-    template <typename OutputT>
-    __device__ __forceinline__ void BlockedToStriped(
-        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        Int2Type<true>  /*time_slicing*/)
-    {
-        InputT temp_items[ITEMS_PER_THREAD];
-
-        #pragma unroll
-        for (int SLICE = 0; SLICE < TIME_SLICES; SLICE++)
-        {
-            const int SLICE_OFFSET  = SLICE * TIME_SLICED_ITEMS;
-            const int SLICE_OOB     = SLICE_OFFSET + TIME_SLICED_ITEMS;
-
-            CTA_SYNC();
-
-            if (warp_id == SLICE)
-            {
-                #pragma unroll
-                for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-                {
-                    int item_offset = (lane_id * ITEMS_PER_THREAD) + ITEM;
-                    if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-                    temp_storage.buff[item_offset] = input_items[ITEM];
-                }
-            }
-
-            CTA_SYNC();
-
-            #pragma unroll
-            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-            {
-                // Read a strip of items
-                const int STRIP_OFFSET  = ITEM * BLOCK_THREADS;
-                const int STRIP_OOB     = STRIP_OFFSET + BLOCK_THREADS;
-
-                if ((SLICE_OFFSET < STRIP_OOB) && (SLICE_OOB > STRIP_OFFSET))
-                {
-                    int item_offset = STRIP_OFFSET + linear_tid - SLICE_OFFSET;
-                    if ((item_offset >= 0) && (item_offset < TIME_SLICED_ITEMS))
-                    {
-                        if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-                        temp_items[ITEM] = temp_storage.buff[item_offset];
-                    }
-                }
-            }
-        }
-
-        // Copy
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            output_items[ITEM] = temp_items[ITEM];
-        }
-    }
-
-
-    /**
-     * Transposes data items from <em>blocked</em> arrangement to <em>warp-striped</em> arrangement. Specialized for no timeslicing
-     */
-    template <typename OutputT>
-    __device__ __forceinline__ void BlockedToWarpStriped(
-        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        Int2Type<false> /*time_slicing*/)
-    {
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            int item_offset = warp_offset + ITEM + (lane_id * ITEMS_PER_THREAD);
-            if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-            temp_storage.buff[item_offset] = input_items[ITEM];
-        }
-
-        WARP_SYNC(0xffffffff);
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            int item_offset = warp_offset + (ITEM * WARP_TIME_SLICED_THREADS) + lane_id;
-            if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-            output_items[ITEM] = temp_storage.buff[item_offset];
-        }
-    }
-
-    /**
-     * Transposes data items from <em>blocked</em> arrangement to <em>warp-striped</em> arrangement. Specialized for warp-timeslicing
-     */
-    template <typename OutputT>
-    __device__ __forceinline__ void BlockedToWarpStriped(
-        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        Int2Type<true>  /*time_slicing*/)
-    {
-        if (warp_id == 0)
-        {
-            #pragma unroll
-            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-            {
-                int item_offset = ITEM + (lane_id * ITEMS_PER_THREAD);
-                if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-                temp_storage.buff[item_offset] = input_items[ITEM];
-            }
-
-            WARP_SYNC(0xffffffff);
-
-            #pragma unroll
-            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-            {
-                int item_offset = (ITEM * WARP_TIME_SLICED_THREADS) + lane_id;
-                if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-                output_items[ITEM] = temp_storage.buff[item_offset];
-            }
-        }
-
-        #pragma unroll
-        for (unsigned int SLICE = 1; SLICE < TIME_SLICES; ++SLICE)
-        {
-            CTA_SYNC();
-
-            if (warp_id == SLICE)
-            {
-                #pragma unroll
-                for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-                {
-                    int item_offset = ITEM + (lane_id * ITEMS_PER_THREAD);
-                    if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-                    temp_storage.buff[item_offset] = input_items[ITEM];
-                }
-
-                WARP_SYNC(0xffffffff);
-
-                #pragma unroll
-                for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-                {
-                    int item_offset = (ITEM * WARP_TIME_SLICED_THREADS) + lane_id;
-                    if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-                    output_items[ITEM] = temp_storage.buff[item_offset];
-                }
-            }
-        }
-    }
-
-
-    /**
-     * Transposes data items from <em>striped</em> arrangement to <em>blocked</em> arrangement.  Specialized for no timeslicing.
-     */
-    template <typename OutputT>
-    __device__ __forceinline__ void StripedToBlocked(
-        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        Int2Type<false> /*time_slicing*/)
-    {
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            int item_offset = int(ITEM * BLOCK_THREADS) + linear_tid;
-            if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-            temp_storage.buff[item_offset] = input_items[ITEM];
-        }
-
-        CTA_SYNC();
-
-        // No timeslicing
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            int item_offset = (linear_tid * ITEMS_PER_THREAD) + ITEM;
-            if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-            output_items[ITEM] = temp_storage.buff[item_offset];
-        }
-    }
-
-
-    /**
-     * Transposes data items from <em>striped</em> arrangement to <em>blocked</em> arrangement.  Specialized for warp-timeslicing.
-     */
-    template <typename OutputT>
-    __device__ __forceinline__ void StripedToBlocked(
-        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        Int2Type<true>  /*time_slicing*/)
-    {
-        // Warp time-slicing
-        InputT temp_items[ITEMS_PER_THREAD];
-
-        #pragma unroll
-        for (int SLICE = 0; SLICE < TIME_SLICES; SLICE++)
-        {
-            const int SLICE_OFFSET  = SLICE * TIME_SLICED_ITEMS;
-            const int SLICE_OOB     = SLICE_OFFSET + TIME_SLICED_ITEMS;
-
-            CTA_SYNC();
-
-            #pragma unroll
-            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-            {
-                // Write a strip of items
-                const int STRIP_OFFSET  = ITEM * BLOCK_THREADS;
-                const int STRIP_OOB     = STRIP_OFFSET + BLOCK_THREADS;
-
-                if ((SLICE_OFFSET < STRIP_OOB) && (SLICE_OOB > STRIP_OFFSET))
-                {
-                    int item_offset = STRIP_OFFSET + linear_tid - SLICE_OFFSET;
-                    if ((item_offset >= 0) && (item_offset < TIME_SLICED_ITEMS))
-                    {
-                        if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-                        temp_storage.buff[item_offset] = input_items[ITEM];
-                    }
-                }
-            }
-
-            CTA_SYNC();
-
-            if (warp_id == SLICE)
-            {
-                #pragma unroll
-                for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-                {
-                    int item_offset = (lane_id * ITEMS_PER_THREAD) + ITEM;
-                    if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-                    temp_items[ITEM] = temp_storage.buff[item_offset];
-                }
-            }
-        }
-
-        // Copy
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            output_items[ITEM] = temp_items[ITEM];
-        }
-    }
-
-
-    /**
-     * Transposes data items from <em>warp-striped</em> arrangement to <em>blocked</em> arrangement.  Specialized for no timeslicing
-     */
-    template <typename OutputT>
-    __device__ __forceinline__ void WarpStripedToBlocked(
-        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        Int2Type<false> /*time_slicing*/)
-    {
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            int item_offset = warp_offset + (ITEM * WARP_TIME_SLICED_THREADS) + lane_id;
-            if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-            temp_storage.buff[item_offset] = input_items[ITEM];
-        }
-
-        WARP_SYNC(0xffffffff);
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            int item_offset = warp_offset + ITEM + (lane_id * ITEMS_PER_THREAD);
-            if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-            output_items[ITEM] = temp_storage.buff[item_offset];
-        }
-    }
-
-
-    /**
-     * Transposes data items from <em>warp-striped</em> arrangement to <em>blocked</em> arrangement.  Specialized for warp-timeslicing
-     */
-    template <typename OutputT>
-    __device__ __forceinline__ void WarpStripedToBlocked(
-        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        Int2Type<true>  /*time_slicing*/)
-    {
-        #pragma unroll
-        for (unsigned int SLICE = 0; SLICE < TIME_SLICES; ++SLICE)
-        {
-            CTA_SYNC();
-
-            if (warp_id == SLICE)
-            {
-                #pragma unroll
-                for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-                {
-                    int item_offset = (ITEM * WARP_TIME_SLICED_THREADS) + lane_id;
-                    if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-                    temp_storage.buff[item_offset] = input_items[ITEM];
-                }
-
-                WARP_SYNC(0xffffffff);
-
-                #pragma unroll
-                for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-                {
-                    int item_offset = ITEM + (lane_id * ITEMS_PER_THREAD);
-                    if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-                    output_items[ITEM] = temp_storage.buff[item_offset];
-                }
-            }
-        }
-    }
-
-
-    /**
-     * Exchanges data items annotated by rank into <em>blocked</em> arrangement.  Specialized for no timeslicing.
-     */
-    template <typename OutputT, typename OffsetT>
-    __device__ __forceinline__ void ScatterToBlocked(
-        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        OffsetT         ranks[ITEMS_PER_THREAD],    ///< [in] Corresponding scatter ranks
-        Int2Type<false> /*time_slicing*/)
-    {
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            int item_offset = ranks[ITEM];
-            if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
-            temp_storage.buff[item_offset] = input_items[ITEM];
-        }
-
-        CTA_SYNC();
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            int item_offset = (linear_tid * ITEMS_PER_THREAD) + ITEM;
-            if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
-            output_items[ITEM] = temp_storage.buff[item_offset];
-        }
-    }
-
-    /**
-     * Exchanges data items annotated by rank into <em>blocked</em> arrangement.  Specialized for warp-timeslicing.
-     */
-    template <typename OutputT, typename OffsetT>
-    __device__ __forceinline__ void ScatterToBlocked(
-        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        OffsetT         ranks[ITEMS_PER_THREAD],    ///< [in] Corresponding scatter ranks
-        Int2Type<true>  /*time_slicing*/)
-    {
-        InputT temp_items[ITEMS_PER_THREAD];
-
-        #pragma unroll
-        for (int SLICE = 0; SLICE < TIME_SLICES; SLICE++)
-        {
-            CTA_SYNC();
-
-            const int SLICE_OFFSET = TIME_SLICED_ITEMS * SLICE;
-
-            #pragma unroll
-            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-            {
-                int item_offset = ranks[ITEM] - SLICE_OFFSET;
-                if ((item_offset >= 0) && (item_offset < WARP_TIME_SLICED_ITEMS))
-                {
-                    if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
-                    temp_storage.buff[item_offset] = input_items[ITEM];
-                }
-            }
-
-            CTA_SYNC();
-
-            if (warp_id == SLICE)
-            {
-                #pragma unroll
-                for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-                {
-                    int item_offset = (lane_id * ITEMS_PER_THREAD) + ITEM;
-                    if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
-                    temp_items[ITEM] = temp_storage.buff[item_offset];
-                }
-            }
-        }
-
-        // Copy
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            output_items[ITEM] = temp_items[ITEM];
-        }
-    }
-
-
-    /**
-     * Exchanges data items annotated by rank into <em>striped</em> arrangement.  Specialized for no timeslicing.
-     */
-    template <typename OutputT, typename OffsetT>
-    __device__ __forceinline__ void ScatterToStriped(
-        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        OffsetT         ranks[ITEMS_PER_THREAD],    ///< [in] Corresponding scatter ranks
-        Int2Type<false> /*time_slicing*/)
-    {
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            int item_offset = ranks[ITEM];
-            if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
-            temp_storage.buff[item_offset] = input_items[ITEM];
-        }
-
-        CTA_SYNC();
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            int item_offset = int(ITEM * BLOCK_THREADS) + linear_tid;
-            if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
-            output_items[ITEM] = temp_storage.buff[item_offset];
-        }
-    }
-
-
-    /**
-     * Exchanges data items annotated by rank into <em>striped</em> arrangement.  Specialized for warp-timeslicing.
-     */
-    template <typename OutputT, typename OffsetT>
-    __device__ __forceinline__ void ScatterToStriped(
-        InputT          input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        OutputT         output_items[ITEMS_PER_THREAD],     ///< [out] Items to exchange, converting between <em>blocked</em> and <em>striped</em> arrangements.
-        OffsetT         ranks[ITEMS_PER_THREAD],    ///< [in] Corresponding scatter ranks
-        Int2Type<true> /*time_slicing*/)
-    {
-        InputT temp_items[ITEMS_PER_THREAD];
-
-        #pragma unroll
-        for (int SLICE = 0; SLICE < TIME_SLICES; SLICE++)
-        {
-            const int SLICE_OFFSET  = SLICE * TIME_SLICED_ITEMS;
-            const int SLICE_OOB     = SLICE_OFFSET + TIME_SLICED_ITEMS;
-
-            CTA_SYNC();
-
-            #pragma unroll
-            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-            {
-                int item_offset = ranks[ITEM] - SLICE_OFFSET;
-                if ((item_offset >= 0) && (item_offset < WARP_TIME_SLICED_ITEMS))
-                {
-                    if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
-                    temp_storage.buff[item_offset] = input_items[ITEM];
-                }
-            }
-
-            CTA_SYNC();
-
-            #pragma unroll
-            for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-            {
-                // Read a strip of items
-                const int STRIP_OFFSET  = ITEM * BLOCK_THREADS;
-                const int STRIP_OOB     = STRIP_OFFSET + BLOCK_THREADS;
-
-                if ((SLICE_OFFSET < STRIP_OOB) && (SLICE_OOB > STRIP_OFFSET))
-                {
-                    int item_offset = STRIP_OFFSET + linear_tid - SLICE_OFFSET;
-                    if ((item_offset >= 0) && (item_offset < TIME_SLICED_ITEMS))
-                    {
-                        if (INSERT_PADDING) item_offset += item_offset >> LOG_SMEM_BANKS;
-                        temp_items[ITEM] = temp_storage.buff[item_offset];
-                    }
-                }
-            }
-        }
-
-        // Copy
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            output_items[ITEM] = temp_items[ITEM];
-        }
-    }
-
-
-public:
-
-    /******************************************************************//**
-     * \name Collective constructors
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
-     */
-    __device__ __forceinline__ BlockExchange()
-    :
-        temp_storage(PrivateStorage()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z)),
-        warp_id((WARPS == 1) ? 0 : linear_tid / WARP_THREADS),
-        lane_id(LaneId()),
-        warp_offset(warp_id * WARP_TIME_SLICED_ITEMS)
-    {}
-
-
-    /**
-     * \brief Collective constructor using the specified memory allocation as temporary storage.
-     */
-    __device__ __forceinline__ BlockExchange(
-        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z)),
-        lane_id(LaneId()),
-        warp_id((WARPS == 1) ? 0 : linear_tid / WARP_THREADS),
-        warp_offset(warp_id * WARP_TIME_SLICED_ITEMS)
-    {}
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Structured exchanges
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Transposes data items from <em>striped</em> arrangement to <em>blocked</em> arrangement.
-     *
-     * \par
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates the conversion from a "striped" to a "blocked" arrangement
-     * of 512 integer items partitioned across 128 threads where each thread owns 4 items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_exchange.cuh>
-     *
-     * __global__ void ExampleKernel(int *d_data, ...)
-     * {
-     *     // Specialize BlockExchange for a 1D block of 128 threads owning 4 integer items each
-     *     typedef cub::BlockExchange<int, 128, 4> BlockExchange;
-     *
-     *     // Allocate shared memory for BlockExchange
-     *     __shared__ typename BlockExchange::TempStorage temp_storage;
-     *
-     *     // Load a tile of ordered data into a striped arrangement across block threads
-     *     int thread_data[4];
-     *     cub::LoadDirectStriped<128>(threadIdx.x, d_data, thread_data);
-     *
-     *     // Collectively exchange data into a blocked arrangement across threads
-     *     BlockExchange(temp_storage).StripedToBlocked(thread_data, thread_data);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of striped input \p thread_data across the block of threads is
-     * <tt>{ [0,128,256,384], [1,129,257,385], ..., [127,255,383,511] }</tt> after loading from device-accessible memory.
-     * The corresponding output \p thread_data in those threads will be
-     * <tt>{ [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }</tt>.
-     *
-     */
-    template <typename OutputT>
-    __device__ __forceinline__ void StripedToBlocked(
-        InputT      input_items[ITEMS_PER_THREAD],    ///< [in] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-        OutputT     output_items[ITEMS_PER_THREAD])   ///< [out] Items from exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-    {
-        StripedToBlocked(input_items, output_items, Int2Type<WARP_TIME_SLICING>());
-    }
-
-
-    /**
-     * \brief Transposes data items from <em>blocked</em> arrangement to <em>striped</em> arrangement.
-     *
-     * \par
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates the conversion from a "blocked" to a "striped" arrangement
-     * of 512 integer items partitioned across 128 threads where each thread owns 4 items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_exchange.cuh>
-     *
-     * __global__ void ExampleKernel(int *d_data, ...)
-     * {
-     *     // Specialize BlockExchange for a 1D block of 128 threads owning 4 integer items each
-     *     typedef cub::BlockExchange<int, 128, 4> BlockExchange;
-     *
-     *     // Allocate shared memory for BlockExchange
-     *     __shared__ typename BlockExchange::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Collectively exchange data into a striped arrangement across threads
-     *     BlockExchange(temp_storage).BlockedToStriped(thread_data, thread_data);
-     *
-     *     // Store data striped across block threads into an ordered tile
-     *     cub::StoreDirectStriped<STORE_DEFAULT, 128>(threadIdx.x, d_data, thread_data);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of blocked input \p thread_data across the block of threads is
-     * <tt>{ [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }</tt>.
-     * The corresponding output \p thread_data in those threads will be
-     * <tt>{ [0,128,256,384], [1,129,257,385], ..., [127,255,383,511] }</tt> in
-     * preparation for storing to device-accessible memory.
-     *
-     */
-    template <typename OutputT>
-    __device__ __forceinline__ void BlockedToStriped(
-        InputT      input_items[ITEMS_PER_THREAD],    ///< [in] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-        OutputT     output_items[ITEMS_PER_THREAD])   ///< [out] Items from exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-    {
-        BlockedToStriped(input_items, output_items, Int2Type<WARP_TIME_SLICING>());
-    }
-
-
-
-    /**
-     * \brief Transposes data items from <em>warp-striped</em> arrangement to <em>blocked</em> arrangement.
-     *
-     * \par
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates the conversion from a "warp-striped" to a "blocked" arrangement
-     * of 512 integer items partitioned across 128 threads where each thread owns 4 items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_exchange.cuh>
-     *
-     * __global__ void ExampleKernel(int *d_data, ...)
-     * {
-     *     // Specialize BlockExchange for a 1D block of 128 threads owning 4 integer items each
-     *     typedef cub::BlockExchange<int, 128, 4> BlockExchange;
-     *
-     *     // Allocate shared memory for BlockExchange
-     *     __shared__ typename BlockExchange::TempStorage temp_storage;
-     *
-     *     // Load a tile of ordered data into a warp-striped arrangement across warp threads
-     *     int thread_data[4];
-     *     cub::LoadSWarptriped<LOAD_DEFAULT>(threadIdx.x, d_data, thread_data);
-     *
-     *     // Collectively exchange data into a blocked arrangement across threads
-     *     BlockExchange(temp_storage).WarpStripedToBlocked(thread_data);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of warp-striped input \p thread_data across the block of threads is
-     * <tt>{ [0,32,64,96], [1,33,65,97], [2,34,66,98], ..., [415,447,479,511] }</tt>
-     * after loading from device-accessible memory.  (The first 128 items are striped across
-     * the first warp of 32 threads, the second 128 items are striped across the second warp, etc.)
-     * The corresponding output \p thread_data in those threads will be
-     * <tt>{ [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }</tt>.
-     *
-     */
-    template <typename OutputT>
-    __device__ __forceinline__ void WarpStripedToBlocked(
-        InputT      input_items[ITEMS_PER_THREAD],    ///< [in] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-        OutputT     output_items[ITEMS_PER_THREAD])   ///< [out] Items from exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-    {
-        WarpStripedToBlocked(input_items, output_items, Int2Type<WARP_TIME_SLICING>());
-    }
-
-
-
-    /**
-     * \brief Transposes data items from <em>blocked</em> arrangement to <em>warp-striped</em> arrangement.
-     *
-     * \par
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates the conversion from a "blocked" to a "warp-striped" arrangement
-     * of 512 integer items partitioned across 128 threads where each thread owns 4 items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_exchange.cuh>
-     *
-     * __global__ void ExampleKernel(int *d_data, ...)
-     * {
-     *     // Specialize BlockExchange for a 1D block of 128 threads owning 4 integer items each
-     *     typedef cub::BlockExchange<int, 128, 4> BlockExchange;
-     *
-     *     // Allocate shared memory for BlockExchange
-     *     __shared__ typename BlockExchange::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Collectively exchange data into a warp-striped arrangement across threads
-     *     BlockExchange(temp_storage).BlockedToWarpStriped(thread_data, thread_data);
-     *
-     *     // Store data striped across warp threads into an ordered tile
-     *     cub::StoreDirectStriped<STORE_DEFAULT, 128>(threadIdx.x, d_data, thread_data);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of blocked input \p thread_data across the block of threads is
-     * <tt>{ [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }</tt>.
-     * The corresponding output \p thread_data in those threads will be
-     * <tt>{ [0,32,64,96], [1,33,65,97], [2,34,66,98], ..., [415,447,479,511] }</tt>
-     * in preparation for storing to device-accessible memory. (The first 128 items are striped across
-     * the first warp of 32 threads, the second 128 items are striped across the second warp, etc.)
-     *
-     */
-    template <typename OutputT>
-    __device__ __forceinline__ void BlockedToWarpStriped(
-        InputT      input_items[ITEMS_PER_THREAD],    ///< [in] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-        OutputT     output_items[ITEMS_PER_THREAD])   ///< [out] Items from exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-    {
-        BlockedToWarpStriped(input_items, output_items, Int2Type<WARP_TIME_SLICING>());
-    }
-
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Scatter exchanges
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Exchanges data items annotated by rank into <em>blocked</em> arrangement.
-     *
-     * \par
-     * - \smemreuse
-     *
-     * \tparam OffsetT                              <b>[inferred]</b> Signed integer type for local offsets
-     */
-    template <typename OutputT, typename OffsetT>
-    __device__ __forceinline__ void ScatterToBlocked(
-        InputT      input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-        OutputT     output_items[ITEMS_PER_THREAD],     ///< [out] Items from exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-        OffsetT     ranks[ITEMS_PER_THREAD])            ///< [in] Corresponding scatter ranks
-    {
-        ScatterToBlocked(input_items, output_items, ranks, Int2Type<WARP_TIME_SLICING>());
-    }
-
-
-
-    /**
-     * \brief Exchanges data items annotated by rank into <em>striped</em> arrangement.
-     *
-     * \par
-     * - \smemreuse
-     *
-     * \tparam OffsetT                              <b>[inferred]</b> Signed integer type for local offsets
-     */
-    template <typename OutputT, typename OffsetT>
-    __device__ __forceinline__ void ScatterToStriped(
-        InputT      input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-        OutputT     output_items[ITEMS_PER_THREAD],     ///< [out] Items from exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-        OffsetT     ranks[ITEMS_PER_THREAD])            ///< [in] Corresponding scatter ranks
-    {
-        ScatterToStriped(input_items, output_items, ranks, Int2Type<WARP_TIME_SLICING>());
-    }
-
-
-
-    /**
-     * \brief Exchanges data items annotated by rank into <em>striped</em> arrangement.  Items with rank -1 are not exchanged.
-     *
-     * \par
-     * - \smemreuse
-     *
-     * \tparam OffsetT                              <b>[inferred]</b> Signed integer type for local offsets
-     */
-    template <typename OutputT, typename OffsetT>
-    __device__ __forceinline__ void ScatterToStripedGuarded(
-        InputT      input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-        OutputT     output_items[ITEMS_PER_THREAD],     ///< [out] Items from exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-        OffsetT     ranks[ITEMS_PER_THREAD])            ///< [in] Corresponding scatter ranks
-    {
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            int item_offset = ranks[ITEM];
-            if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
-            if (ranks[ITEM] >= 0)
-                temp_storage.buff[item_offset] = input_items[ITEM];
-        }
-
-        CTA_SYNC();
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            int item_offset = int(ITEM * BLOCK_THREADS) + linear_tid;
-            if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
-            output_items[ITEM] = temp_storage.buff[item_offset];
-        }
-    }
-
-
-
-
-    /**
-     * \brief Exchanges valid data items annotated by rank into <em>striped</em> arrangement.
-     *
-     * \par
-     * - \smemreuse
-     *
-     * \tparam OffsetT                              <b>[inferred]</b> Signed integer type for local offsets
-     * \tparam ValidFlag                            <b>[inferred]</b> FlagT type denoting which items are valid
-     */
-    template <typename OutputT, typename OffsetT, typename ValidFlag>
-    __device__ __forceinline__ void ScatterToStripedFlagged(
-        InputT      input_items[ITEMS_PER_THREAD],      ///< [in] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-        OutputT     output_items[ITEMS_PER_THREAD],     ///< [out] Items from exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-        OffsetT     ranks[ITEMS_PER_THREAD],            ///< [in] Corresponding scatter ranks
-        ValidFlag   is_valid[ITEMS_PER_THREAD])         ///< [in] Corresponding flag denoting item validity
-    {
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            int item_offset = ranks[ITEM];
-            if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
-            if (is_valid[ITEM])
-                temp_storage.buff[item_offset] = input_items[ITEM];
-        }
-
-        CTA_SYNC();
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            int item_offset = int(ITEM * BLOCK_THREADS) + linear_tid;
-            if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
-            output_items[ITEM] = temp_storage.buff[item_offset];
-        }
-    }
-
-
-    //@}  end member group
-
-
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-
-    __device__ __forceinline__ void StripedToBlocked(
-        InputT      items[ITEMS_PER_THREAD])   ///< [in-out] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-    {
-        StripedToBlocked(items, items);
-    }
-
-    __device__ __forceinline__ void BlockedToStriped(
-        InputT      items[ITEMS_PER_THREAD])   ///< [in-out] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-    {
-        BlockedToStriped(items, items);
-    }
-
-    __device__ __forceinline__ void WarpStripedToBlocked(
-        InputT      items[ITEMS_PER_THREAD])    ///< [in-out] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-    {
-        WarpStripedToBlocked(items, items);
-    }
-
-    __device__ __forceinline__ void BlockedToWarpStriped(
-        InputT      items[ITEMS_PER_THREAD])    ///< [in-out] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-    {
-        BlockedToWarpStriped(items, items);
-    }
-
-    template <typename OffsetT>
-    __device__ __forceinline__ void ScatterToBlocked(
-        InputT      items[ITEMS_PER_THREAD],    ///< [in-out] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-        OffsetT     ranks[ITEMS_PER_THREAD])    ///< [in] Corresponding scatter ranks
-    {
-        ScatterToBlocked(items, items, ranks);
-    }
-
-    template <typename OffsetT>
-    __device__ __forceinline__ void ScatterToStriped(
-        InputT      items[ITEMS_PER_THREAD],    ///< [in-out] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-        OffsetT     ranks[ITEMS_PER_THREAD])    ///< [in] Corresponding scatter ranks
-    {
-        ScatterToStriped(items, items, ranks);
-    }
-
-    template <typename OffsetT>
-    __device__ __forceinline__ void ScatterToStripedGuarded(
-        InputT      items[ITEMS_PER_THREAD],    ///< [in-out] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-        OffsetT     ranks[ITEMS_PER_THREAD])    ///< [in] Corresponding scatter ranks
-    {
-        ScatterToStripedGuarded(items, items, ranks);
-    }
-
-    template <typename OffsetT, typename ValidFlag>
-    __device__ __forceinline__ void ScatterToStripedFlagged(
-        InputT      items[ITEMS_PER_THREAD],        ///< [in-out] Items to exchange, converting between <em>striped</em> and <em>blocked</em> arrangements.
-        OffsetT     ranks[ITEMS_PER_THREAD],        ///< [in] Corresponding scatter ranks
-        ValidFlag   is_valid[ITEMS_PER_THREAD])     ///< [in] Corresponding flag denoting item validity
-    {
-        ScatterToStriped(items, items, ranks, is_valid);
-    }
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-};
-
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-
-template <
-    typename    T,
-    int         ITEMS_PER_THREAD,
-    int         LOGICAL_WARP_THREADS    = CUB_PTX_WARP_THREADS,
-    int         PTX_ARCH                = CUB_PTX_ARCH>
-class WarpExchange
-{
-private:
-
-    /******************************************************************************
-     * Constants
-     ******************************************************************************/
-
-    /// Constants
-    enum
-    {
-        // Whether the logical warp size and the PTX warp size coincide
-        IS_ARCH_WARP = (LOGICAL_WARP_THREADS == CUB_WARP_THREADS(PTX_ARCH)),
-
-        WARP_ITEMS                  = (ITEMS_PER_THREAD * LOGICAL_WARP_THREADS) + 1,
-
-        LOG_SMEM_BANKS              = CUB_LOG_SMEM_BANKS(PTX_ARCH),
-        SMEM_BANKS                  = 1 << LOG_SMEM_BANKS,
-
-        // Insert padding if the number of items per thread is a power of two and > 4 (otherwise we can typically use 128b loads)
-        INSERT_PADDING              = (ITEMS_PER_THREAD > 4) && (PowerOfTwo<ITEMS_PER_THREAD>::VALUE),
-        PADDING_ITEMS               = (INSERT_PADDING) ? (WARP_ITEMS >> LOG_SMEM_BANKS) : 0,
-    };
-
-    /******************************************************************************
-     * Type definitions
-     ******************************************************************************/
-
-    /// Shared memory storage layout type
-    struct _TempStorage
-    {
-        T buff[WARP_ITEMS + PADDING_ITEMS];
-    };
-
-public:
-
-    /// \smemstorage{WarpExchange}
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-private:
-
-
-    /******************************************************************************
-     * Thread fields
-     ******************************************************************************/
-
-    _TempStorage    &temp_storage;
-    int             lane_id;
-
-public:
-
-    /******************************************************************************
-     * Construction
-     ******************************************************************************/
-
-    /// Constructor
-    __device__ __forceinline__ WarpExchange(
-        TempStorage &temp_storage)
-    :
-        temp_storage(temp_storage.Alias()),
-        lane_id(IS_ARCH_WARP ?
-            LaneId() :
-            LaneId() % LOGICAL_WARP_THREADS)
-    {}
-
-
-    /******************************************************************************
-     * Interface
-     ******************************************************************************/
-
-    /**
-     * \brief Exchanges valid data items annotated by rank into <em>striped</em> arrangement.
-     *
-     * \par
-     * - \smemreuse
-     *
-     * \tparam OffsetT                              <b>[inferred]</b> Signed integer type for local offsets
-     */
-    template <typename OffsetT>
-    __device__ __forceinline__ void ScatterToStriped(
-        T               items[ITEMS_PER_THREAD],        ///< [in-out] Items to exchange
-        OffsetT         ranks[ITEMS_PER_THREAD])        ///< [in] Corresponding scatter ranks
-    {
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            if (INSERT_PADDING) ranks[ITEM] = SHR_ADD(ranks[ITEM], LOG_SMEM_BANKS, ranks[ITEM]);
-            temp_storage.buff[ranks[ITEM]] = items[ITEM];
-        }
-
-        WARP_SYNC(0xffffffff);
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        {
-            int item_offset = (ITEM * LOGICAL_WARP_THREADS) + lane_id;
-            if (INSERT_PADDING) item_offset = SHR_ADD(item_offset, LOG_SMEM_BANKS, item_offset);
-            items[ITEM] = temp_storage.buff[item_offset];
-        }
-    }
-
-};
-
-
-
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/block/block_histogram.cuh b/pfsimulator/third_party/cub/block/block_histogram.cuh
deleted file mode 100644
index b7cb9700e..000000000
--- a/pfsimulator/third_party/cub/block/block_histogram.cuh
+++ /dev/null
@@ -1,415 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * The cub::BlockHistogram class provides [<em>collective</em>](index.html#sec0) methods for constructing block-wide histograms from data samples partitioned across a CUDA thread block.
- */
-
-#pragma once
-
-#include "specializations/block_histogram_sort.cuh"
-#include "specializations/block_histogram_atomic.cuh"
-#include "../util_ptx.cuh"
-#include "../util_arch.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/******************************************************************************
- * Algorithmic variants
- ******************************************************************************/
-
-/**
- * \brief BlockHistogramAlgorithm enumerates alternative algorithms for the parallel construction of block-wide histograms.
- */
-enum BlockHistogramAlgorithm
-{
-
-    /**
-     * \par Overview
-     * Sorting followed by differentiation.  Execution is comprised of two phases:
-     * -# Sort the data using efficient radix sort
-     * -# Look for "runs" of same-valued keys by detecting discontinuities; the run-lengths are histogram bin counts.
-     *
-     * \par Performance Considerations
-     * Delivers consistent throughput regardless of sample bin distribution.
-     */
-    BLOCK_HISTO_SORT,
-
-
-    /**
-     * \par Overview
-     * Use atomic addition to update byte counts directly
-     *
-     * \par Performance Considerations
-     * Performance is strongly tied to the hardware implementation of atomic
-     * addition, and may be significantly degraded for non uniformly-random
-     * input distributions where many concurrent updates are likely to be
-     * made to the same bin counter.
-     */
-    BLOCK_HISTO_ATOMIC,
-};
-
-
-
-/******************************************************************************
- * Block histogram
- ******************************************************************************/
-
-
-/**
- * \brief The BlockHistogram class provides [<em>collective</em>](index.html#sec0) methods for constructing block-wide histograms from data samples partitioned across a CUDA thread block. ![](histogram_logo.png)
- * \ingroup BlockModule
- *
- * \tparam T                    The sample type being histogrammed (must be castable to an integer bin identifier)
- * \tparam BLOCK_DIM_X          The thread block length in threads along the X dimension
- * \tparam ITEMS_PER_THREAD     The number of items per thread
- * \tparam BINS                 The number bins within the histogram
- * \tparam ALGORITHM            <b>[optional]</b> cub::BlockHistogramAlgorithm enumerator specifying the underlying algorithm to use (default: cub::BLOCK_HISTO_SORT)
- * \tparam BLOCK_DIM_Y          <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
- * \tparam BLOCK_DIM_Z          <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
- * \tparam PTX_ARCH             <b>[optional]</b> \ptxversion
- *
- * \par Overview
- * - A <a href="http://en.wikipedia.org/wiki/Histogram"><em>histogram</em></a>
- *   counts the number of observations that fall into each of the disjoint categories (known as <em>bins</em>).
- * - BlockHistogram can be optionally specialized to use different algorithms:
- *   -# <b>cub::BLOCK_HISTO_SORT</b>.  Sorting followed by differentiation. [More...](\ref cub::BlockHistogramAlgorithm)
- *   -# <b>cub::BLOCK_HISTO_ATOMIC</b>.  Use atomic addition to update byte counts directly. [More...](\ref cub::BlockHistogramAlgorithm)
- *
- * \par Performance Considerations
- * - \granularity
- *
- * \par A Simple Example
- * \blockcollective{BlockHistogram}
- * \par
- * The code snippet below illustrates a 256-bin histogram of 512 integer samples that
- * are partitioned across 128 threads where each thread owns 4 samples.
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/block/block_histogram.cuh>
- *
- * __global__ void ExampleKernel(...)
- * {
- *     // Specialize a 256-bin BlockHistogram type for a 1D block of 128 threads having 4 character samples each
- *     typedef cub::BlockHistogram<unsigned char, 128, 4, 256> BlockHistogram;
- *
- *     // Allocate shared memory for BlockHistogram
- *     __shared__ typename BlockHistogram::TempStorage temp_storage;
- *
- *     // Allocate shared memory for block-wide histogram bin counts
- *     __shared__ unsigned int smem_histogram[256];
- *
- *     // Obtain input samples per thread
- *     unsigned char data[4];
- *     ...
- *
- *     // Compute the block-wide histogram
- *     BlockHistogram(temp_storage).Histogram(data, smem_histogram);
- *
- * \endcode
- *
- * \par Performance and Usage Considerations
- * - The histogram output can be constructed in shared or device-accessible memory
- * - See cub::BlockHistogramAlgorithm for performance details regarding algorithmic alternatives
- *
- */
-template <
-    typename                T,
-    int                     BLOCK_DIM_X,
-    int                     ITEMS_PER_THREAD,
-    int                     BINS,
-    BlockHistogramAlgorithm ALGORITHM           = BLOCK_HISTO_SORT,
-    int                     BLOCK_DIM_Y         = 1,
-    int                     BLOCK_DIM_Z         = 1,
-    int                     PTX_ARCH            = CUB_PTX_ARCH>
-class BlockHistogram
-{
-private:
-
-    /******************************************************************************
-     * Constants and type definitions
-     ******************************************************************************/
-
-    /// Constants
-    enum
-    {
-        /// The thread block size in threads
-        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-    };
-
-    /**
-     * Ensure the template parameterization meets the requirements of the
-     * targeted device architecture.  BLOCK_HISTO_ATOMIC can only be used
-     * on version SM120 or later.  Otherwise BLOCK_HISTO_SORT is used
-     * regardless.
-     */
-    static const BlockHistogramAlgorithm SAFE_ALGORITHM =
-        ((ALGORITHM == BLOCK_HISTO_ATOMIC) && (PTX_ARCH < 120)) ?
-            BLOCK_HISTO_SORT :
-            ALGORITHM;
-
-    /// Internal specialization.
-    typedef typename If<(SAFE_ALGORITHM == BLOCK_HISTO_SORT),
-        BlockHistogramSort<T, BLOCK_DIM_X, ITEMS_PER_THREAD, BINS, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH>,
-        BlockHistogramAtomic<BINS> >::Type InternalBlockHistogram;
-
-    /// Shared memory storage layout type for BlockHistogram
-    typedef typename InternalBlockHistogram::TempStorage _TempStorage;
-
-
-    /******************************************************************************
-     * Thread fields
-     ******************************************************************************/
-
-    /// Shared storage reference
-    _TempStorage &temp_storage;
-
-    /// Linear thread-id
-    unsigned int linear_tid;
-
-
-    /******************************************************************************
-     * Utility methods
-     ******************************************************************************/
-
-    /// Internal storage allocator
-    __device__ __forceinline__ _TempStorage& PrivateStorage()
-    {
-        __shared__ _TempStorage private_storage;
-        return private_storage;
-    }
-
-
-public:
-
-    /// \smemstorage{BlockHistogram}
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    /******************************************************************//**
-     * \name Collective constructors
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
-     */
-    __device__ __forceinline__ BlockHistogram()
-    :
-        temp_storage(PrivateStorage()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    /**
-     * \brief Collective constructor using the specified memory allocation as temporary storage.
-     */
-    __device__ __forceinline__ BlockHistogram(
-        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Histogram operations
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Initialize the shared histogram counters to zero.
-     *
-     * \par Snippet
-     * The code snippet below illustrates a the initialization and update of a
-     * histogram of 512 integer samples that are partitioned across 128 threads
-     * where each thread owns 4 samples.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_histogram.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize a 256-bin BlockHistogram type for a 1D block of 128 threads having 4 character samples each
-     *     typedef cub::BlockHistogram<unsigned char, 128, 4, 256> BlockHistogram;
-     *
-     *     // Allocate shared memory for BlockHistogram
-     *     __shared__ typename BlockHistogram::TempStorage temp_storage;
-     *
-     *     // Allocate shared memory for block-wide histogram bin counts
-     *     __shared__ unsigned int smem_histogram[256];
-     *
-     *     // Obtain input samples per thread
-     *     unsigned char thread_samples[4];
-     *     ...
-     *
-     *     // Initialize the block-wide histogram
-     *     BlockHistogram(temp_storage).InitHistogram(smem_histogram);
-     *
-     *     // Update the block-wide histogram
-     *     BlockHistogram(temp_storage).Composite(thread_samples, smem_histogram);
-     *
-     * \endcode
-     *
-     * \tparam CounterT              <b>[inferred]</b> Histogram counter type
-     */
-    template <typename CounterT     >
-    __device__ __forceinline__ void InitHistogram(CounterT      histogram[BINS])
-    {
-        // Initialize histogram bin counts to zeros
-        int histo_offset = 0;
-
-        #pragma unroll
-        for(; histo_offset + BLOCK_THREADS <= BINS; histo_offset += BLOCK_THREADS)
-        {
-            histogram[histo_offset + linear_tid] = 0;
-        }
-        // Finish up with guarded initialization if necessary
-        if ((BINS % BLOCK_THREADS != 0) && (histo_offset + linear_tid < BINS))
-        {
-            histogram[histo_offset + linear_tid] = 0;
-        }
-    }
-
-
-    /**
-     * \brief Constructs a block-wide histogram in shared/device-accessible memory.  Each thread contributes an array of input elements.
-     *
-     * \par
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a 256-bin histogram of 512 integer samples that
-     * are partitioned across 128 threads where each thread owns 4 samples.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_histogram.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize a 256-bin BlockHistogram type for a 1D block of 128 threads having 4 character samples each
-     *     typedef cub::BlockHistogram<unsigned char, 128, 4, 256> BlockHistogram;
-     *
-     *     // Allocate shared memory for BlockHistogram
-     *     __shared__ typename BlockHistogram::TempStorage temp_storage;
-     *
-     *     // Allocate shared memory for block-wide histogram bin counts
-     *     __shared__ unsigned int smem_histogram[256];
-     *
-     *     // Obtain input samples per thread
-     *     unsigned char thread_samples[4];
-     *     ...
-     *
-     *     // Compute the block-wide histogram
-     *     BlockHistogram(temp_storage).Histogram(thread_samples, smem_histogram);
-     *
-     * \endcode
-     *
-     * \tparam CounterT              <b>[inferred]</b> Histogram counter type
-     */
-    template <
-        typename            CounterT     >
-    __device__ __forceinline__ void Histogram(
-        T                   (&items)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input values to histogram
-        CounterT             histogram[BINS])                ///< [out] Reference to shared/device-accessible memory histogram
-    {
-        // Initialize histogram bin counts to zeros
-        InitHistogram(histogram);
-
-        CTA_SYNC();
-
-        // Composite the histogram
-        InternalBlockHistogram(temp_storage).Composite(items, histogram);
-    }
-
-
-
-    /**
-     * \brief Updates an existing block-wide histogram in shared/device-accessible memory.  Each thread composites an array of input elements.
-     *
-     * \par
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a the initialization and update of a
-     * histogram of 512 integer samples that are partitioned across 128 threads
-     * where each thread owns 4 samples.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_histogram.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize a 256-bin BlockHistogram type for a 1D block of 128 threads having 4 character samples each
-     *     typedef cub::BlockHistogram<unsigned char, 128, 4, 256> BlockHistogram;
-     *
-     *     // Allocate shared memory for BlockHistogram
-     *     __shared__ typename BlockHistogram::TempStorage temp_storage;
-     *
-     *     // Allocate shared memory for block-wide histogram bin counts
-     *     __shared__ unsigned int smem_histogram[256];
-     *
-     *     // Obtain input samples per thread
-     *     unsigned char thread_samples[4];
-     *     ...
-     *
-     *     // Initialize the block-wide histogram
-     *     BlockHistogram(temp_storage).InitHistogram(smem_histogram);
-     *
-     *     // Update the block-wide histogram
-     *     BlockHistogram(temp_storage).Composite(thread_samples, smem_histogram);
-     *
-     * \endcode
-     *
-     * \tparam CounterT              <b>[inferred]</b> Histogram counter type
-     */
-    template <
-        typename            CounterT     >
-    __device__ __forceinline__ void Composite(
-        T                   (&items)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input values to histogram
-        CounterT             histogram[BINS])                 ///< [out] Reference to shared/device-accessible memory histogram
-    {
-        InternalBlockHistogram(temp_storage).Composite(items, histogram);
-    }
-
-};
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/block/block_load.cuh b/pfsimulator/third_party/cub/block/block_load.cuh
deleted file mode 100644
index 217f52123..000000000
--- a/pfsimulator/third_party/cub/block/block_load.cuh
+++ /dev/null
@@ -1,1241 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2016, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Operations for reading linear tiles of data into the CUDA thread block.
- */
-
-#pragma once
-
-#include <iterator>
-
-#include "block_exchange.cuh"
-#include "../iterator/cache_modified_input_iterator.cuh"
-#include "../util_ptx.cuh"
-#include "../util_macro.cuh"
-#include "../util_type.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \addtogroup UtilIo
- * @{
- */
-
-
-/******************************************************************//**
- * \name Blocked arrangement I/O (direct)
- *********************************************************************/
-//@{
-
-
-/**
- * \brief Load a linear segment of items into a blocked arrangement across the thread block.
- *
- * \blocked
- *
- * \tparam T                    <b>[inferred]</b> The data type to load.
- * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
- * \tparam InputIteratorT       <b>[inferred]</b> The random-access iterator type for input \iterator.
- */
-template <
-    typename        InputT,
-    int             ITEMS_PER_THREAD,
-    typename        InputIteratorT>
-__device__ __forceinline__ void LoadDirectBlocked(
-    int             linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
-    InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
-    InputT          (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load
-{
-    InputIteratorT thread_itr = block_itr + (linear_tid * ITEMS_PER_THREAD);
-
-    // Load directly in thread-blocked order
-    #pragma unroll
-    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-    {
-        items[ITEM] = thread_itr[ITEM];
-    }
-}
-
-
-/**
- * \brief Load a linear segment of items into a blocked arrangement across the thread block, guarded by range.
- *
- * \blocked
- *
- * \tparam T                    <b>[inferred]</b> The data type to load.
- * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
- * \tparam InputIteratorT       <b>[inferred]</b> The random-access iterator type for input \iterator.
- */
-template <
-    typename        InputT,
-    int             ITEMS_PER_THREAD,
-    typename        InputIteratorT>
-__device__ __forceinline__ void LoadDirectBlocked(
-    int             linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
-    InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
-    InputT          (&items)[ITEMS_PER_THREAD], ///< [out] Data to load
-    int             valid_items)                ///< [in] Number of valid items to load
-{
-    InputIteratorT thread_itr = block_itr + (linear_tid * ITEMS_PER_THREAD);
-
-    #pragma unroll
-    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-    {
-        if ((linear_tid * ITEMS_PER_THREAD) + ITEM < valid_items)
-        {
-            items[ITEM] = thread_itr[ITEM];
-        }
-    }
-}
-
-
-/**
- * \brief Load a linear segment of items into a blocked arrangement across the thread block, guarded by range, with a fall-back assignment of out-of-bound elements..
- *
- * \blocked
- *
- * \tparam T                    <b>[inferred]</b> The data type to load.
- * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
- * \tparam InputIteratorT       <b>[inferred]</b> The random-access iterator type for input \iterator.
- */
-template <
-    typename        InputT,
-    typename        DefaultT,
-    int             ITEMS_PER_THREAD,
-    typename        InputIteratorT>
-__device__ __forceinline__ void LoadDirectBlocked(
-    int             linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
-    InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
-    InputT          (&items)[ITEMS_PER_THREAD], ///< [out] Data to load
-    int             valid_items,                ///< [in] Number of valid items to load
-    DefaultT        oob_default)                ///< [in] Default value to assign out-of-bound items
-{
-    #pragma unroll
-    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        items[ITEM] = oob_default;
-
-    LoadDirectBlocked(linear_tid, block_itr, items, valid_items);
-}
-
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-/**
- * Internal implementation for load vectorization
- */
-template <
-    CacheLoadModifier   MODIFIER,
-    typename            T,
-    int                 ITEMS_PER_THREAD>
-__device__ __forceinline__ void InternalLoadDirectBlockedVectorized(
-    int    linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
-    T      *block_ptr,                 ///< [in] Input pointer for loading from
-    T      (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load
-{
-    // Biggest memory access word that T is a whole multiple of
-    typedef typename UnitWord<T>::DeviceWord DeviceWord;
-
-    enum
-    {
-        TOTAL_WORDS = sizeof(items) / sizeof(DeviceWord),
-
-        VECTOR_SIZE = (TOTAL_WORDS % 4 == 0) ?
-            4 :
-            (TOTAL_WORDS % 2 == 0) ?
-                2 :
-                1,
-
-        VECTORS_PER_THREAD = TOTAL_WORDS / VECTOR_SIZE,
-    };
-
-    // Vector type
-    typedef typename CubVector<DeviceWord, VECTOR_SIZE>::Type Vector;
-
-    // Vector items
-    Vector vec_items[VECTORS_PER_THREAD];
-
-    // Aliased input ptr
-    Vector* vec_ptr = reinterpret_cast<Vector*>(block_ptr) + (linear_tid * VECTORS_PER_THREAD);
-
-    // Load directly in thread-blocked order
-    #pragma unroll
-    for (int ITEM = 0; ITEM < VECTORS_PER_THREAD; ITEM++)
-    {
-        vec_items[ITEM] = ThreadLoad<MODIFIER>(vec_ptr + ITEM);
-    }
-
-    // Copy
-    #pragma unroll
-    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-    {
-        items[ITEM] = *(reinterpret_cast<T*>(vec_items) + ITEM);
-    }
-}
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-/**
- * \brief Load a linear segment of items into a blocked arrangement across the thread block.
- *
- * \blocked
- *
- * The input offset (\p block_ptr + \p block_offset) must be quad-item aligned
- *
- * The following conditions will prevent vectorization and loading will fall back to cub::BLOCK_LOAD_DIRECT:
- *   - \p ITEMS_PER_THREAD is odd
- *   - The data type \p T is not a built-in primitive or CUDA vector type (e.g., \p short, \p int2, \p double, \p float2, etc.)
- *
- * \tparam T                    <b>[inferred]</b> The data type to load.
- * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
- */
-template <
-    typename        T,
-    int             ITEMS_PER_THREAD>
-__device__ __forceinline__ void LoadDirectBlockedVectorized(
-    int linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
-    T   *block_ptr,                 ///< [in] Input pointer for loading from
-    T   (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load
-{
-    InternalLoadDirectBlockedVectorized<LOAD_DEFAULT>(linear_tid, block_ptr, items);
-}
-
-
-//@}  end member group
-/******************************************************************//**
- * \name Striped arrangement I/O (direct)
- *********************************************************************/
-//@{
-
-
-/**
- * \brief Load a linear segment of items into a striped arrangement across the thread block.
- *
- * \striped
- *
- * \tparam BLOCK_THREADS        The thread block size in threads
- * \tparam T                    <b>[inferred]</b> The data type to load.
- * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
- * \tparam InputIteratorT       <b>[inferred]</b> The random-access iterator type for input \iterator.
- */
-template <
-    int             BLOCK_THREADS,
-    typename        InputT,
-    int             ITEMS_PER_THREAD,
-    typename        InputIteratorT>
-__device__ __forceinline__ void LoadDirectStriped(
-    int             linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
-    InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
-    InputT          (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load
-{
-    InputIteratorT thread_itr = block_itr + linear_tid;
-
-    #pragma unroll
-    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-    {
-        items[ITEM] = thread_itr[ITEM * BLOCK_THREADS];
-    }
-}
-
-
-/**
- * \brief Load a linear segment of items into a striped arrangement across the thread block, guarded by range
- *
- * \striped
- *
- * \tparam BLOCK_THREADS        The thread block size in threads
- * \tparam T                    <b>[inferred]</b> The data type to load.
- * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
- * \tparam InputIteratorT       <b>[inferred]</b> The random-access iterator type for input \iterator.
- */
-template <
-    int             BLOCK_THREADS,
-    typename        InputT,
-    int             ITEMS_PER_THREAD,
-    typename        InputIteratorT>
-__device__ __forceinline__ void LoadDirectStriped(
-    int             linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
-    InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
-    InputT          (&items)[ITEMS_PER_THREAD], ///< [out] Data to load
-    int             valid_items)                ///< [in] Number of valid items to load
-{
-    InputIteratorT thread_itr = block_itr + linear_tid;
-
-    #pragma unroll
-    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-    {
-        if (linear_tid + (ITEM * BLOCK_THREADS) < valid_items)
-        {
-            items[ITEM] = thread_itr[ITEM * BLOCK_THREADS];
-        }
-    }
-}
-
-
-/**
- * \brief Load a linear segment of items into a striped arrangement across the thread block, guarded by range, with a fall-back assignment of out-of-bound elements.
- *
- * \striped
- *
- * \tparam BLOCK_THREADS        The thread block size in threads
- * \tparam T                    <b>[inferred]</b> The data type to load.
- * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
- * \tparam InputIteratorT       <b>[inferred]</b> The random-access iterator type for input \iterator.
- */
-template <
-    int             BLOCK_THREADS,
-    typename        InputT,
-    typename        DefaultT,
-    int             ITEMS_PER_THREAD,
-    typename        InputIteratorT>
-__device__ __forceinline__ void LoadDirectStriped(
-    int             linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
-    InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
-    InputT          (&items)[ITEMS_PER_THREAD], ///< [out] Data to load
-    int             valid_items,                ///< [in] Number of valid items to load
-    DefaultT        oob_default)                ///< [in] Default value to assign out-of-bound items
-{
-    #pragma unroll
-    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        items[ITEM] = oob_default;
-
-    LoadDirectStriped<BLOCK_THREADS>(linear_tid, block_itr, items, valid_items);
-}
-
-
-
-//@}  end member group
-/******************************************************************//**
- * \name Warp-striped arrangement I/O (direct)
- *********************************************************************/
-//@{
-
-
-/**
- * \brief Load a linear segment of items into a warp-striped arrangement across the thread block.
- *
- * \warpstriped
- *
- * \par Usage Considerations
- * The number of threads in the thread block must be a multiple of the architecture's warp size.
- *
- * \tparam T                    <b>[inferred]</b> The data type to load.
- * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
- * \tparam InputIteratorT       <b>[inferred]</b> The random-access iterator type for input \iterator.
- */
-template <
-    typename        InputT,
-    int             ITEMS_PER_THREAD,
-    typename        InputIteratorT>
-__device__ __forceinline__ void LoadDirectWarpStriped(
-    int             linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
-    InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
-    InputT          (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load
-{
-    int tid                = linear_tid & (CUB_PTX_WARP_THREADS - 1);
-    int wid                = linear_tid >> CUB_PTX_LOG_WARP_THREADS;
-    int warp_offset        = wid * CUB_PTX_WARP_THREADS * ITEMS_PER_THREAD;
-
-    InputIteratorT thread_itr = block_itr + warp_offset + tid ;
-
-    // Load directly in warp-striped order
-    #pragma unroll
-    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-    {
-        items[ITEM] = thread_itr[(ITEM * CUB_PTX_WARP_THREADS)];
-    }
-}
-
-
-/**
- * \brief Load a linear segment of items into a warp-striped arrangement across the thread block, guarded by range
- *
- * \warpstriped
- *
- * \par Usage Considerations
- * The number of threads in the thread block must be a multiple of the architecture's warp size.
- *
- * \tparam T                    <b>[inferred]</b> The data type to load.
- * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
- * \tparam InputIteratorT        <b>[inferred]</b> The random-access iterator type for input \iterator.
- */
-template <
-    typename        InputT,
-    int             ITEMS_PER_THREAD,
-    typename        InputIteratorT>
-__device__ __forceinline__ void LoadDirectWarpStriped(
-    int             linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
-    InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
-    InputT          (&items)[ITEMS_PER_THREAD], ///< [out] Data to load
-    int             valid_items)                ///< [in] Number of valid items to load
-{
-    int tid                = linear_tid & (CUB_PTX_WARP_THREADS - 1);
-    int wid                = linear_tid >> CUB_PTX_LOG_WARP_THREADS;
-    int warp_offset        = wid * CUB_PTX_WARP_THREADS * ITEMS_PER_THREAD;
-
-    InputIteratorT thread_itr = block_itr + warp_offset + tid ;
-
-    // Load directly in warp-striped order
-    #pragma unroll
-    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-    {
-        if (warp_offset + tid + (ITEM * CUB_PTX_WARP_THREADS) < valid_items)
-        {
-            items[ITEM] = thread_itr[(ITEM * CUB_PTX_WARP_THREADS)];
-        }
-    }
-}
-
-
-/**
- * \brief Load a linear segment of items into a warp-striped arrangement across the thread block, guarded by range, with a fall-back assignment of out-of-bound elements.
- *
- * \warpstriped
- *
- * \par Usage Considerations
- * The number of threads in the thread block must be a multiple of the architecture's warp size.
- *
- * \tparam T                    <b>[inferred]</b> The data type to load.
- * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
- * \tparam InputIteratorT        <b>[inferred]</b> The random-access iterator type for input \iterator.
- */
-template <
-    typename        InputT,
-    typename        DefaultT,
-    int             ITEMS_PER_THREAD,
-    typename        InputIteratorT>
-__device__ __forceinline__ void LoadDirectWarpStriped(
-    int             linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
-    InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
-    InputT          (&items)[ITEMS_PER_THREAD], ///< [out] Data to load
-    int             valid_items,                ///< [in] Number of valid items to load
-    DefaultT        oob_default)                ///< [in] Default value to assign out-of-bound items
-{
-    // Load directly in warp-striped order
-    #pragma unroll
-    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-        items[ITEM] = oob_default;
-
-    LoadDirectWarpStriped(linear_tid, block_itr, items, valid_items);
-}
-
-
-
-//@}  end member group
-
-/** @} */       // end group UtilIo
-
-
-
-//-----------------------------------------------------------------------------
-// Generic BlockLoad abstraction
-//-----------------------------------------------------------------------------
-
-/**
- * \brief cub::BlockLoadAlgorithm enumerates alternative algorithms for cub::BlockLoad to read a linear segment of data from memory into a blocked arrangement across a CUDA thread block.
- */
-
-/**
- * \brief cub::BlockLoadAlgorithm enumerates alternative algorithms for cub::BlockLoad to read a linear segment of data from memory into a blocked arrangement across a CUDA thread block.
- */
-enum BlockLoadAlgorithm
-{
-    /**
-     * \par Overview
-     *
-     * A [<em>blocked arrangement</em>](index.html#sec5sec3) of data is read
-     * directly from memory.
-     *
-     * \par Performance Considerations
-     * - The utilization of memory transactions (coalescing) decreases as the
-     *   access stride between threads increases (i.e., the number items per thread).
-     */
-    BLOCK_LOAD_DIRECT,
-
-    /**
-     * \par Overview
-     *
-     * A [<em>blocked arrangement</em>](index.html#sec5sec3) of data is read
-     * from memory using CUDA's built-in vectorized loads as a coalescing optimization.
-     * For example, <tt>ld.global.v4.s32</tt> instructions will be generated
-     * when \p T = \p int and \p ITEMS_PER_THREAD % 4 == 0.
-     *
-     * \par Performance Considerations
-     * - The utilization of memory transactions (coalescing) remains high until the the
-     *   access stride between threads (i.e., the number items per thread) exceeds the
-     *   maximum vector load width (typically 4 items or 64B, whichever is lower).
-     * - The following conditions will prevent vectorization and loading will fall back to cub::BLOCK_LOAD_DIRECT:
-     *   - \p ITEMS_PER_THREAD is odd
-     *   - The \p InputIteratorTis not a simple pointer type
-     *   - The block input offset is not quadword-aligned
-     *   - The data type \p T is not a built-in primitive or CUDA vector type (e.g., \p short, \p int2, \p double, \p float2, etc.)
-     */
-    BLOCK_LOAD_VECTORIZE,
-
-    /**
-     * \par Overview
-     *
-     * A [<em>striped arrangement</em>](index.html#sec5sec3) of data is read
-     * efficiently from memory and then locally transposed into a
-     * [<em>blocked arrangement</em>](index.html#sec5sec3).
-     *
-     * \par Performance Considerations
-     * - The utilization of memory transactions (coalescing) remains high regardless
-     *   of items loaded per thread.
-     * - The local reordering incurs slightly longer latencies and throughput than the
-     *   direct cub::BLOCK_LOAD_DIRECT and cub::BLOCK_LOAD_VECTORIZE alternatives.
-     */
-    BLOCK_LOAD_TRANSPOSE,
-
-
-    /**
-     * \par Overview
-     *
-     * A [<em>warp-striped arrangement</em>](index.html#sec5sec3) of data is
-     * read efficiently from memory and then locally transposed into a
-     * [<em>blocked arrangement</em>](index.html#sec5sec3).
-     *
-     * \par Usage Considerations
-     * - BLOCK_THREADS must be a multiple of WARP_THREADS
-     *
-     * \par Performance Considerations
-     * - The utilization of memory transactions (coalescing) remains high regardless
-     *   of items loaded per thread.
-     * - The local reordering incurs slightly larger latencies than the
-     *   direct cub::BLOCK_LOAD_DIRECT and cub::BLOCK_LOAD_VECTORIZE alternatives.
-     * - Provisions more shared storage, but incurs smaller latencies than the
-     *   BLOCK_LOAD_WARP_TRANSPOSE_TIMESLICED alternative.
-     */
-    BLOCK_LOAD_WARP_TRANSPOSE,
-
-
-    /**
-     * \par Overview
-     *
-     * Like \p BLOCK_LOAD_WARP_TRANSPOSE, a [<em>warp-striped arrangement</em>](index.html#sec5sec3)
-     * of data is read directly from memory and then is locally transposed into a
-     * [<em>blocked arrangement</em>](index.html#sec5sec3). To reduce the shared memory
-     * requirement, only one warp's worth of shared memory is provisioned and is
-     * subsequently time-sliced among warps.
-     *
-     * \par Usage Considerations
-     * - BLOCK_THREADS must be a multiple of WARP_THREADS
-     *
-     * \par Performance Considerations
-     * - The utilization of memory transactions (coalescing) remains high regardless
-     *   of items loaded per thread.
-     * - Provisions less shared memory temporary storage, but incurs larger
-     *   latencies than the BLOCK_LOAD_WARP_TRANSPOSE alternative.
-     */
-    BLOCK_LOAD_WARP_TRANSPOSE_TIMESLICED,
-};
-
-
-/**
- * \brief The BlockLoad class provides [<em>collective</em>](index.html#sec0) data movement methods for loading a linear segment of items from memory into a [<em>blocked arrangement</em>](index.html#sec5sec3) across a CUDA thread block.  ![](block_load_logo.png)
- * \ingroup BlockModule
- * \ingroup UtilIo
- *
- * \tparam InputT               The data type to read into (which must be convertible from the input iterator's value type).
- * \tparam BLOCK_DIM_X          The thread block length in threads along the X dimension
- * \tparam ITEMS_PER_THREAD     The number of consecutive items partitioned onto each thread.
- * \tparam ALGORITHM            <b>[optional]</b> cub::BlockLoadAlgorithm tuning policy.  default: cub::BLOCK_LOAD_DIRECT.
- * \tparam WARP_TIME_SLICING    <b>[optional]</b> Whether or not only one warp's worth of shared memory should be allocated and time-sliced among block-warps during any load-related data transpositions (versus each warp having its own storage). (default: false)
- * \tparam BLOCK_DIM_Y          <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
- * \tparam BLOCK_DIM_Z          <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
- * \tparam PTX_ARCH             <b>[optional]</b> \ptxversion
- *
- * \par Overview
- * - The BlockLoad class provides a single data movement abstraction that can be specialized
- *   to implement different cub::BlockLoadAlgorithm strategies.  This facilitates different
- *   performance policies for different architectures, data types, granularity sizes, etc.
- * - BlockLoad can be optionally specialized by different data movement strategies:
- *   -# <b>cub::BLOCK_LOAD_DIRECT</b>.  A [<em>blocked arrangement</em>](index.html#sec5sec3)
- *      of data is read directly from memory.  [More...](\ref cub::BlockLoadAlgorithm)
- *   -# <b>cub::BLOCK_LOAD_VECTORIZE</b>.  A [<em>blocked arrangement</em>](index.html#sec5sec3)
- *      of data is read directly from memory using CUDA's built-in vectorized loads as a
- *      coalescing optimization.    [More...](\ref cub::BlockLoadAlgorithm)
- *   -# <b>cub::BLOCK_LOAD_TRANSPOSE</b>.  A [<em>striped arrangement</em>](index.html#sec5sec3)
- *      of data is read directly from memory and is then locally transposed into a
- *      [<em>blocked arrangement</em>](index.html#sec5sec3).  [More...](\ref cub::BlockLoadAlgorithm)
- *   -# <b>cub::BLOCK_LOAD_WARP_TRANSPOSE</b>.  A [<em>warp-striped arrangement</em>](index.html#sec5sec3)
- *      of data is read directly from memory and is then locally transposed into a
- *      [<em>blocked arrangement</em>](index.html#sec5sec3).  [More...](\ref cub::BlockLoadAlgorithm)
- *   -# <b>cub::BLOCK_LOAD_WARP_TRANSPOSE_TIMESLICED,</b>.  A [<em>warp-striped arrangement</em>](index.html#sec5sec3)
- *      of data is read directly from memory and is then locally transposed into a
- *      [<em>blocked arrangement</em>](index.html#sec5sec3) one warp at a time.  [More...](\ref cub::BlockLoadAlgorithm)
- * - \rowmajor
- *
- * \par A Simple Example
- * \blockcollective{BlockLoad}
- * \par
- * The code snippet below illustrates the loading of a linear
- * segment of 512 integers into a "blocked" arrangement across 128 threads where each
- * thread owns 4 consecutive items.  The load is specialized for \p BLOCK_LOAD_WARP_TRANSPOSE,
- * meaning memory references are efficiently coalesced using a warp-striped access
- * pattern (after which items are locally reordered among threads).
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/block/block_load.cuh>
- *
- * __global__ void ExampleKernel(int *d_data, ...)
- * {
- *     // Specialize BlockLoad for a 1D block of 128 threads owning 4 integer items each
- *     typedef cub::BlockLoad<int, 128, 4, BLOCK_LOAD_WARP_TRANSPOSE> BlockLoad;
- *
- *     // Allocate shared memory for BlockLoad
- *     __shared__ typename BlockLoad::TempStorage temp_storage;
- *
- *     // Load a segment of consecutive items that are blocked across threads
- *     int thread_data[4];
- *     BlockLoad(temp_storage).Load(d_data, thread_data);
- *
- * \endcode
- * \par
- * Suppose the input \p d_data is <tt>0, 1, 2, 3, 4, 5, ...</tt>.
- * The set of \p thread_data across the block of threads in those threads will be
- * <tt>{ [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }</tt>.
- *
- */
-template <
-    typename            InputT,
-    int                 BLOCK_DIM_X,
-    int                 ITEMS_PER_THREAD,
-    BlockLoadAlgorithm  ALGORITHM           = BLOCK_LOAD_DIRECT,
-    int                 BLOCK_DIM_Y         = 1,
-    int                 BLOCK_DIM_Z         = 1,
-    int                 PTX_ARCH            = CUB_PTX_ARCH>
-class BlockLoad
-{
-private:
-
-    /******************************************************************************
-     * Constants and typed definitions
-     ******************************************************************************/
-
-    /// Constants
-    enum
-    {
-        /// The thread block size in threads
-        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-    };
-
-
-    /******************************************************************************
-     * Algorithmic variants
-     ******************************************************************************/
-
-    /// Load helper
-    template <BlockLoadAlgorithm _POLICY, int DUMMY>
-    struct LoadInternal;
-
-
-    /**
-     * BLOCK_LOAD_DIRECT specialization of load helper
-     */
-    template <int DUMMY>
-    struct LoadInternal<BLOCK_LOAD_DIRECT, DUMMY>
-    {
-        /// Shared memory storage layout type
-        typedef NullType TempStorage;
-
-        /// Linear thread-id
-        int linear_tid;
-
-        /// Constructor
-        __device__ __forceinline__ LoadInternal(
-            TempStorage &/*temp_storage*/,
-            int linear_tid)
-        :
-            linear_tid(linear_tid)
-        {}
-
-        /// Load a linear segment of items from memory
-        template <typename InputIteratorT>
-        __device__ __forceinline__ void Load(
-            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
-            InputT          (&items)[ITEMS_PER_THREAD])     ///< [out] Data to load
-        {
-            LoadDirectBlocked(linear_tid, block_itr, items);
-        }
-
-        /// Load a linear segment of items from memory, guarded by range
-        template <typename InputIteratorT>
-        __device__ __forceinline__ void Load(
-            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
-            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
-            int             valid_items)                    ///< [in] Number of valid items to load
-        {
-            LoadDirectBlocked(linear_tid, block_itr, items, valid_items);
-        }
-
-        /// Load a linear segment of items from memory, guarded by range, with a fall-back assignment of out-of-bound elements
-        template <typename InputIteratorT, typename DefaultT>
-        __device__ __forceinline__ void Load(
-            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
-            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
-            int             valid_items,                    ///< [in] Number of valid items to load
-            DefaultT        oob_default)                    ///< [in] Default value to assign out-of-bound items
-        {
-            LoadDirectBlocked(linear_tid, block_itr, items, valid_items, oob_default);
-        }
-
-    };
-
-
-    /**
-     * BLOCK_LOAD_VECTORIZE specialization of load helper
-     */
-    template <int DUMMY>
-    struct LoadInternal<BLOCK_LOAD_VECTORIZE, DUMMY>
-    {
-        /// Shared memory storage layout type
-        typedef NullType TempStorage;
-
-        /// Linear thread-id
-        int linear_tid;
-
-        /// Constructor
-        __device__ __forceinline__ LoadInternal(
-            TempStorage &/*temp_storage*/,
-            int linear_tid)
-        :
-            linear_tid(linear_tid)
-        {}
-
-        /// Load a linear segment of items from memory, specialized for native pointer types (attempts vectorization)
-        template <typename InputIteratorT>
-        __device__ __forceinline__ void Load(
-            InputT               *block_ptr,                     ///< [in] The thread block's base input iterator for loading from
-            InputT               (&items)[ITEMS_PER_THREAD])     ///< [out] Data to load
-        {
-            InternalLoadDirectBlockedVectorized<LOAD_DEFAULT>(linear_tid, block_ptr, items);
-        }
-
-        /// Load a linear segment of items from memory, specialized for native pointer types (attempts vectorization)
-        template <typename InputIteratorT>
-        __device__ __forceinline__ void Load(
-            const InputT         *block_ptr,                     ///< [in] The thread block's base input iterator for loading from
-            InputT               (&items)[ITEMS_PER_THREAD])     ///< [out] Data to load
-        {
-            InternalLoadDirectBlockedVectorized<LOAD_DEFAULT>(linear_tid, block_ptr, items);
-        }
-
-        /// Load a linear segment of items from memory, specialized for native pointer types (attempts vectorization)
-        template <
-            CacheLoadModifier   MODIFIER,
-            typename            ValueType,
-            typename            OffsetT>
-        __device__ __forceinline__ void Load(
-            CacheModifiedInputIterator<MODIFIER, ValueType, OffsetT>    block_itr,                      ///< [in] The thread block's base input iterator for loading from
-            InputT                                                     (&items)[ITEMS_PER_THREAD])     ///< [out] Data to load
-        {
-            InternalLoadDirectBlockedVectorized<MODIFIER>(linear_tid, block_itr.ptr, items);
-        }
-
-        /// Load a linear segment of items from memory, specialized for opaque input iterators (skips vectorization)
-        template <typename _InputIteratorT>
-        __device__ __forceinline__ void Load(
-            _InputIteratorT   block_itr,                    ///< [in] The thread block's base input iterator for loading from
-            InputT           (&items)[ITEMS_PER_THREAD])   ///< [out] Data to load
-        {
-            LoadDirectBlocked(linear_tid, block_itr, items);
-        }
-
-        /// Load a linear segment of items from memory, guarded by range (skips vectorization)
-        template <typename InputIteratorT>
-        __device__ __forceinline__ void Load(
-            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
-            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
-            int             valid_items)                    ///< [in] Number of valid items to load
-        {
-            LoadDirectBlocked(linear_tid, block_itr, items, valid_items);
-        }
-
-        /// Load a linear segment of items from memory, guarded by range, with a fall-back assignment of out-of-bound elements (skips vectorization)
-        template <typename InputIteratorT, typename DefaultT>
-        __device__ __forceinline__ void Load(
-            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
-            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
-            int             valid_items,                    ///< [in] Number of valid items to load
-            DefaultT          oob_default)                    ///< [in] Default value to assign out-of-bound items
-        {
-            LoadDirectBlocked(linear_tid, block_itr, items, valid_items, oob_default);
-        }
-
-    };
-
-
-    /**
-     * BLOCK_LOAD_TRANSPOSE specialization of load helper
-     */
-    template <int DUMMY>
-    struct LoadInternal<BLOCK_LOAD_TRANSPOSE, DUMMY>
-    {
-        // BlockExchange utility type for keys
-        typedef BlockExchange<InputT, BLOCK_DIM_X, ITEMS_PER_THREAD, false, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> BlockExchange;
-
-        /// Shared memory storage layout type
-        struct _TempStorage : BlockExchange::TempStorage
-        {};
-
-        /// Alias wrapper allowing storage to be unioned
-        struct TempStorage : Uninitialized<_TempStorage> {};
-
-        /// Thread reference to shared storage
-        _TempStorage &temp_storage;
-
-        /// Linear thread-id
-        int linear_tid;
-
-        /// Constructor
-        __device__ __forceinline__ LoadInternal(
-            TempStorage &temp_storage,
-            int linear_tid)
-        :
-            temp_storage(temp_storage.Alias()),
-            linear_tid(linear_tid)
-        {}
-
-        /// Load a linear segment of items from memory
-        template <typename InputIteratorT>
-        __device__ __forceinline__ void Load(
-            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
-            InputT          (&items)[ITEMS_PER_THREAD])     ///< [out] Data to load{
-        {
-            LoadDirectStriped<BLOCK_THREADS>(linear_tid, block_itr, items);
-            BlockExchange(temp_storage).StripedToBlocked(items, items);
-        }
-
-        /// Load a linear segment of items from memory, guarded by range
-        template <typename InputIteratorT>
-        __device__ __forceinline__ void Load(
-            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
-            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
-            int             valid_items)                    ///< [in] Number of valid items to load
-        {
-            LoadDirectStriped<BLOCK_THREADS>(linear_tid, block_itr, items, valid_items);
-            BlockExchange(temp_storage).StripedToBlocked(items, items);
-        }
-
-        /// Load a linear segment of items from memory, guarded by range, with a fall-back assignment of out-of-bound elements
-        template <typename InputIteratorT, typename DefaultT>
-        __device__ __forceinline__ void Load(
-            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
-            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
-            int             valid_items,                    ///< [in] Number of valid items to load
-            DefaultT        oob_default)                    ///< [in] Default value to assign out-of-bound items
-        {
-            LoadDirectStriped<BLOCK_THREADS>(linear_tid, block_itr, items, valid_items, oob_default);
-            BlockExchange(temp_storage).StripedToBlocked(items, items);
-        }
-
-    };
-
-
-    /**
-     * BLOCK_LOAD_WARP_TRANSPOSE specialization of load helper
-     */
-    template <int DUMMY>
-    struct LoadInternal<BLOCK_LOAD_WARP_TRANSPOSE, DUMMY>
-    {
-        enum
-        {
-            WARP_THREADS = CUB_WARP_THREADS(PTX_ARCH)
-        };
-
-        // Assert BLOCK_THREADS must be a multiple of WARP_THREADS
-        CUB_STATIC_ASSERT((BLOCK_THREADS % WARP_THREADS == 0), "BLOCK_THREADS must be a multiple of WARP_THREADS");
-
-        // BlockExchange utility type for keys
-        typedef BlockExchange<InputT, BLOCK_DIM_X, ITEMS_PER_THREAD, false, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> BlockExchange;
-
-        /// Shared memory storage layout type
-        struct _TempStorage : BlockExchange::TempStorage
-        {};
-
-        /// Alias wrapper allowing storage to be unioned
-        struct TempStorage : Uninitialized<_TempStorage> {};
-
-        /// Thread reference to shared storage
-        _TempStorage &temp_storage;
-
-        /// Linear thread-id
-        int linear_tid;
-
-        /// Constructor
-        __device__ __forceinline__ LoadInternal(
-            TempStorage &temp_storage,
-            int linear_tid)
-        :
-            temp_storage(temp_storage.Alias()),
-            linear_tid(linear_tid)
-        {}
-
-        /// Load a linear segment of items from memory
-        template <typename InputIteratorT>
-        __device__ __forceinline__ void Load(
-            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
-            InputT          (&items)[ITEMS_PER_THREAD])     ///< [out] Data to load{
-        {
-            LoadDirectWarpStriped(linear_tid, block_itr, items);
-            BlockExchange(temp_storage).WarpStripedToBlocked(items, items);
-        }
-
-        /// Load a linear segment of items from memory, guarded by range
-        template <typename InputIteratorT>
-        __device__ __forceinline__ void Load(
-            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
-            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
-            int             valid_items)                    ///< [in] Number of valid items to load
-        {
-            LoadDirectWarpStriped(linear_tid, block_itr, items, valid_items);
-            BlockExchange(temp_storage).WarpStripedToBlocked(items, items);
-        }
-
-
-        /// Load a linear segment of items from memory, guarded by range, with a fall-back assignment of out-of-bound elements
-        template <typename InputIteratorT, typename DefaultT>
-        __device__ __forceinline__ void Load(
-            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
-            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
-            int             valid_items,                    ///< [in] Number of valid items to load
-            DefaultT        oob_default)                    ///< [in] Default value to assign out-of-bound items
-        {
-            LoadDirectWarpStriped(linear_tid, block_itr, items, valid_items, oob_default);
-            BlockExchange(temp_storage).WarpStripedToBlocked(items, items);
-        }
-    };
-
-
-    /**
-     * BLOCK_LOAD_WARP_TRANSPOSE_TIMESLICED specialization of load helper
-     */
-    template <int DUMMY>
-    struct LoadInternal<BLOCK_LOAD_WARP_TRANSPOSE_TIMESLICED, DUMMY>
-    {
-        enum
-        {
-            WARP_THREADS = CUB_WARP_THREADS(PTX_ARCH)
-        };
-
-        // Assert BLOCK_THREADS must be a multiple of WARP_THREADS
-        CUB_STATIC_ASSERT((BLOCK_THREADS % WARP_THREADS == 0), "BLOCK_THREADS must be a multiple of WARP_THREADS");
-
-        // BlockExchange utility type for keys
-        typedef BlockExchange<InputT, BLOCK_DIM_X, ITEMS_PER_THREAD, true, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> BlockExchange;
-
-        /// Shared memory storage layout type
-        struct _TempStorage : BlockExchange::TempStorage
-        {};
-
-        /// Alias wrapper allowing storage to be unioned
-        struct TempStorage : Uninitialized<_TempStorage> {};
-
-        /// Thread reference to shared storage
-        _TempStorage &temp_storage;
-
-        /// Linear thread-id
-        int linear_tid;
-
-        /// Constructor
-        __device__ __forceinline__ LoadInternal(
-            TempStorage &temp_storage,
-            int linear_tid)
-        :
-            temp_storage(temp_storage.Alias()),
-            linear_tid(linear_tid)
-        {}
-
-        /// Load a linear segment of items from memory
-        template <typename InputIteratorT>
-        __device__ __forceinline__ void Load(
-            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
-            InputT          (&items)[ITEMS_PER_THREAD])     ///< [out] Data to load{
-        {
-            LoadDirectWarpStriped(linear_tid, block_itr, items);
-            BlockExchange(temp_storage).WarpStripedToBlocked(items, items);
-        }
-
-        /// Load a linear segment of items from memory, guarded by range
-        template <typename InputIteratorT>
-        __device__ __forceinline__ void Load(
-            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
-            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
-            int             valid_items)                    ///< [in] Number of valid items to load
-        {
-            LoadDirectWarpStriped(linear_tid, block_itr, items, valid_items);
-            BlockExchange(temp_storage).WarpStripedToBlocked(items, items);
-        }
-
-
-        /// Load a linear segment of items from memory, guarded by range, with a fall-back assignment of out-of-bound elements
-        template <typename InputIteratorT, typename DefaultT>
-        __device__ __forceinline__ void Load(
-            InputIteratorT  block_itr,                      ///< [in] The thread block's base input iterator for loading from
-            InputT          (&items)[ITEMS_PER_THREAD],     ///< [out] Data to load
-            int             valid_items,                    ///< [in] Number of valid items to load
-            DefaultT        oob_default)                    ///< [in] Default value to assign out-of-bound items
-        {
-            LoadDirectWarpStriped(linear_tid, block_itr, items, valid_items, oob_default);
-            BlockExchange(temp_storage).WarpStripedToBlocked(items, items);
-        }
-    };
-
-
-    /******************************************************************************
-     * Type definitions
-     ******************************************************************************/
-
-    /// Internal load implementation to use
-    typedef LoadInternal<ALGORITHM, 0> InternalLoad;
-
-
-    /// Shared memory storage layout type
-    typedef typename InternalLoad::TempStorage _TempStorage;
-
-
-    /******************************************************************************
-     * Utility methods
-     ******************************************************************************/
-
-    /// Internal storage allocator
-    __device__ __forceinline__ _TempStorage& PrivateStorage()
-    {
-        __shared__ _TempStorage private_storage;
-        return private_storage;
-    }
-
-
-    /******************************************************************************
-     * Thread fields
-     ******************************************************************************/
-
-    /// Thread reference to shared storage
-    _TempStorage &temp_storage;
-
-    /// Linear thread-id
-    int linear_tid;
-
-public:
-
-    /// \smemstorage{BlockLoad}
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    /******************************************************************//**
-     * \name Collective constructors
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
-     */
-    __device__ __forceinline__ BlockLoad()
-    :
-        temp_storage(PrivateStorage()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    /**
-     * \brief Collective constructor using the specified memory allocation as temporary storage.
-     */
-    __device__ __forceinline__ BlockLoad(
-        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Data movement
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Load a linear segment of items from memory.
-     *
-     * \par
-     * - \blocked
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates the loading of a linear
-     * segment of 512 integers into a "blocked" arrangement across 128 threads where each
-     * thread owns 4 consecutive items.  The load is specialized for \p BLOCK_LOAD_WARP_TRANSPOSE,
-     * meaning memory references are efficiently coalesced using a warp-striped access
-     * pattern (after which items are locally reordered among threads).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_load.cuh>
-     *
-     * __global__ void ExampleKernel(int *d_data, ...)
-     * {
-     *     // Specialize BlockLoad for a 1D block of 128 threads owning 4 integer items each
-     *     typedef cub::BlockLoad<int, 128, 4, BLOCK_LOAD_WARP_TRANSPOSE> BlockLoad;
-     *
-     *     // Allocate shared memory for BlockLoad
-     *     __shared__ typename BlockLoad::TempStorage temp_storage;
-     *
-     *     // Load a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     BlockLoad(temp_storage).Load(d_data, thread_data);
-     *
-     * \endcode
-     * \par
-     * Suppose the input \p d_data is <tt>0, 1, 2, 3, 4, 5, ...</tt>.
-     * The set of \p thread_data across the block of threads in those threads will be
-     * <tt>{ [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }</tt>.
-     *
-     */
-    template <typename InputIteratorT>
-    __device__ __forceinline__ void Load(
-        InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
-        InputT          (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load
-    {
-        InternalLoad(temp_storage, linear_tid).Load(block_itr, items);
-    }
-
-
-    /**
-     * \brief Load a linear segment of items from memory, guarded by range.
-     *
-     * \par
-     * - \blocked
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates the guarded loading of a linear
-     * segment of 512 integers into a "blocked" arrangement across 128 threads where each
-     * thread owns 4 consecutive items.  The load is specialized for \p BLOCK_LOAD_WARP_TRANSPOSE,
-     * meaning memory references are efficiently coalesced using a warp-striped access
-     * pattern (after which items are locally reordered among threads).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_load.cuh>
-     *
-     * __global__ void ExampleKernel(int *d_data, int valid_items, ...)
-     * {
-     *     // Specialize BlockLoad for a 1D block of 128 threads owning 4 integer items each
-     *     typedef cub::BlockLoad<int, 128, 4, BLOCK_LOAD_WARP_TRANSPOSE> BlockLoad;
-     *
-     *     // Allocate shared memory for BlockLoad
-     *     __shared__ typename BlockLoad::TempStorage temp_storage;
-     *
-     *     // Load a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     BlockLoad(temp_storage).Load(d_data, thread_data, valid_items);
-     *
-     * \endcode
-     * \par
-     * Suppose the input \p d_data is <tt>0, 1, 2, 3, 4, 5, 6...</tt> and \p valid_items is \p 5.
-     * The set of \p thread_data across the block of threads in those threads will be
-     * <tt>{ [0,1,2,3], [4,?,?,?], ..., [?,?,?,?] }</tt>, with only the first two threads
-     * being unmasked to load portions of valid data (and other items remaining unassigned).
-     *
-     */
-    template <typename InputIteratorT>
-    __device__ __forceinline__ void Load(
-        InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
-        InputT          (&items)[ITEMS_PER_THREAD], ///< [out] Data to load
-        int             valid_items)                ///< [in] Number of valid items to load
-    {
-        InternalLoad(temp_storage, linear_tid).Load(block_itr, items, valid_items);
-    }
-
-
-    /**
-     * \brief Load a linear segment of items from memory, guarded by range, with a fall-back assignment of out-of-bound elements
-     *
-     * \par
-     * - \blocked
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates the guarded loading of a linear
-     * segment of 512 integers into a "blocked" arrangement across 128 threads where each
-     * thread owns 4 consecutive items.  The load is specialized for \p BLOCK_LOAD_WARP_TRANSPOSE,
-     * meaning memory references are efficiently coalesced using a warp-striped access
-     * pattern (after which items are locally reordered among threads).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_load.cuh>
-     *
-     * __global__ void ExampleKernel(int *d_data, int valid_items, ...)
-     * {
-     *     // Specialize BlockLoad for a 1D block of 128 threads owning 4 integer items each
-     *     typedef cub::BlockLoad<int, 128, 4, BLOCK_LOAD_WARP_TRANSPOSE> BlockLoad;
-     *
-     *     // Allocate shared memory for BlockLoad
-     *     __shared__ typename BlockLoad::TempStorage temp_storage;
-     *
-     *     // Load a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     BlockLoad(temp_storage).Load(d_data, thread_data, valid_items, -1);
-     *
-     * \endcode
-     * \par
-     * Suppose the input \p d_data is <tt>0, 1, 2, 3, 4, 5, 6...</tt>,
-     * \p valid_items is \p 5, and the out-of-bounds default is \p -1.
-     * The set of \p thread_data across the block of threads in those threads will be
-     * <tt>{ [0,1,2,3], [4,-1,-1,-1], ..., [-1,-1,-1,-1] }</tt>, with only the first two threads
-     * being unmasked to load portions of valid data (and other items are assigned \p -1)
-     *
-     */
-    template <typename InputIteratorT, typename DefaultT>
-    __device__ __forceinline__ void Load(
-        InputIteratorT  block_itr,                  ///< [in] The thread block's base input iterator for loading from
-        InputT          (&items)[ITEMS_PER_THREAD], ///< [out] Data to load
-        int             valid_items,                ///< [in] Number of valid items to load
-        DefaultT        oob_default)                ///< [in] Default value to assign out-of-bound items
-    {
-        InternalLoad(temp_storage, linear_tid).Load(block_itr, items, valid_items, oob_default);
-    }
-
-
-    //@}  end member group
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/block/block_radix_rank.cuh b/pfsimulator/third_party/cub/block/block_radix_rank.cuh
deleted file mode 100644
index c26451c66..000000000
--- a/pfsimulator/third_party/cub/block/block_radix_rank.cuh
+++ /dev/null
@@ -1,696 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::BlockRadixRank provides operations for ranking unsigned integer types within a CUDA thread block
- */
-
-#pragma once
-
-#include <stdint.h>
-
-#include "../thread/thread_reduce.cuh"
-#include "../thread/thread_scan.cuh"
-#include "../block/block_scan.cuh"
-#include "../util_ptx.cuh"
-#include "../util_arch.cuh"
-#include "../util_type.cuh"
-#include "../util_namespace.cuh"
-
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \brief BlockRadixRank provides operations for ranking unsigned integer types within a CUDA thread block.
- * \ingroup BlockModule
- *
- * \tparam BLOCK_DIM_X          The thread block length in threads along the X dimension
- * \tparam RADIX_BITS           The number of radix bits per digit place
- * \tparam IS_DESCENDING           Whether or not the sorted-order is high-to-low
- * \tparam MEMOIZE_OUTER_SCAN   <b>[optional]</b> Whether or not to buffer outer raking scan partials to incur fewer shared memory reads at the expense of higher register pressure (default: true for architectures SM35 and newer, false otherwise).  See BlockScanAlgorithm::BLOCK_SCAN_RAKING_MEMOIZE for more details.
- * \tparam INNER_SCAN_ALGORITHM <b>[optional]</b> The cub::BlockScanAlgorithm algorithm to use (default: cub::BLOCK_SCAN_WARP_SCANS)
- * \tparam SMEM_CONFIG          <b>[optional]</b> Shared memory bank mode (default: \p cudaSharedMemBankSizeFourByte)
- * \tparam BLOCK_DIM_Y          <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
- * \tparam BLOCK_DIM_Z          <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
- * \tparam PTX_ARCH             <b>[optional]</b> \ptxversion
- *
- * \par Overview
- * Blah...
- * - Keys must be in a form suitable for radix ranking (i.e., unsigned bits).
- * - \blocked
- *
- * \par Performance Considerations
- * - \granularity
- *
- * \par Examples
- * \par
- * - <b>Example 1:</b> Simple radix rank of 32-bit integer keys
- *      \code
- *      #include <cub/cub.cuh>
- *
- *      template <int BLOCK_THREADS>
- *      __global__ void ExampleKernel(...)
- *      {
- *
- *      \endcode
- */
-template <
-    int                     BLOCK_DIM_X,
-    int                     RADIX_BITS,
-    bool                    IS_DESCENDING,
-    bool                    MEMOIZE_OUTER_SCAN      = (CUB_PTX_ARCH >= 350) ? true : false,
-    BlockScanAlgorithm      INNER_SCAN_ALGORITHM    = BLOCK_SCAN_WARP_SCANS,
-    cudaSharedMemConfig     SMEM_CONFIG             = cudaSharedMemBankSizeFourByte,
-    int                     BLOCK_DIM_Y             = 1,
-    int                     BLOCK_DIM_Z             = 1,
-    int                     PTX_ARCH                = CUB_PTX_ARCH>
-class BlockRadixRank
-{
-private:
-
-    /******************************************************************************
-     * Type definitions and constants
-     ******************************************************************************/
-
-    // Integer type for digit counters (to be packed into words of type PackedCounters)
-    typedef unsigned short DigitCounter;
-
-    // Integer type for packing DigitCounters into columns of shared memory banks
-    typedef typename If<(SMEM_CONFIG == cudaSharedMemBankSizeEightByte),
-        unsigned long long,
-        unsigned int>::Type PackedCounter;
-
-    enum
-    {
-        // The thread block size in threads
-        BLOCK_THREADS               = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-
-        RADIX_DIGITS                = 1 << RADIX_BITS,
-
-        LOG_WARP_THREADS            = CUB_LOG_WARP_THREADS(PTX_ARCH),
-        WARP_THREADS                = 1 << LOG_WARP_THREADS,
-        WARPS                       = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS,
-
-        BYTES_PER_COUNTER           = sizeof(DigitCounter),
-        LOG_BYTES_PER_COUNTER       = Log2<BYTES_PER_COUNTER>::VALUE,
-
-        PACKING_RATIO               = sizeof(PackedCounter) / sizeof(DigitCounter),
-        LOG_PACKING_RATIO           = Log2<PACKING_RATIO>::VALUE,
-
-        LOG_COUNTER_LANES           = CUB_MAX((RADIX_BITS - LOG_PACKING_RATIO), 0),                // Always at least one lane
-        COUNTER_LANES               = 1 << LOG_COUNTER_LANES,
-
-        // The number of packed counters per thread (plus one for padding)
-        PADDED_COUNTER_LANES        = COUNTER_LANES + 1,
-        RAKING_SEGMENT              = PADDED_COUNTER_LANES,
-    };
-
-public:
-
-    enum
-    {
-        /// Number of bin-starting offsets tracked per thread
-        BINS_TRACKED_PER_THREAD = CUB_MAX(1, (RADIX_DIGITS + BLOCK_THREADS - 1) / BLOCK_THREADS),
-    };
-
-private:
-
-
-    /// BlockScan type
-    typedef BlockScan<
-            PackedCounter,
-            BLOCK_DIM_X,
-            INNER_SCAN_ALGORITHM,
-            BLOCK_DIM_Y,
-            BLOCK_DIM_Z,
-            PTX_ARCH>
-        BlockScan;
-
-
-    /// Shared memory storage layout type for BlockRadixRank
-    struct __align__(16) _TempStorage
-    {
-        union Aliasable
-        {
-            DigitCounter            digit_counters[PADDED_COUNTER_LANES][BLOCK_THREADS][PACKING_RATIO];
-            PackedCounter           raking_grid[BLOCK_THREADS][RAKING_SEGMENT];
-
-        } aliasable;
-
-        // Storage for scanning local ranks
-        typename BlockScan::TempStorage block_scan;
-    };
-
-
-    /******************************************************************************
-     * Thread fields
-     ******************************************************************************/
-
-    /// Shared storage reference
-    _TempStorage &temp_storage;
-
-    /// Linear thread-id
-    unsigned int linear_tid;
-
-    /// Copy of raking segment, promoted to registers
-    PackedCounter cached_segment[RAKING_SEGMENT];
-
-
-    /******************************************************************************
-     * Utility methods
-     ******************************************************************************/
-
-    /**
-     * Internal storage allocator
-     */
-    __device__ __forceinline__ _TempStorage& PrivateStorage()
-    {
-        __shared__ _TempStorage private_storage;
-        return private_storage;
-    }
-
-
-    /**
-     * Performs upsweep raking reduction, returning the aggregate
-     */
-    __device__ __forceinline__ PackedCounter Upsweep()
-    {
-        PackedCounter *smem_raking_ptr = temp_storage.aliasable.raking_grid[linear_tid];
-        PackedCounter *raking_ptr;
-
-        if (MEMOIZE_OUTER_SCAN)
-        {
-            // Copy data into registers
-            #pragma unroll
-            for (int i = 0; i < RAKING_SEGMENT; i++)
-            {
-                cached_segment[i] = smem_raking_ptr[i];
-            }
-            raking_ptr = cached_segment;
-        }
-        else
-        {
-            raking_ptr = smem_raking_ptr;
-        }
-
-        return internal::ThreadReduce<RAKING_SEGMENT>(raking_ptr, Sum());
-    }
-
-
-    /// Performs exclusive downsweep raking scan
-    __device__ __forceinline__ void ExclusiveDownsweep(
-        PackedCounter raking_partial)
-    {
-        PackedCounter *smem_raking_ptr = temp_storage.aliasable.raking_grid[linear_tid];
-
-        PackedCounter *raking_ptr = (MEMOIZE_OUTER_SCAN) ?
-            cached_segment :
-            smem_raking_ptr;
-
-        // Exclusive raking downsweep scan
-        internal::ThreadScanExclusive<RAKING_SEGMENT>(raking_ptr, raking_ptr, Sum(), raking_partial);
-
-        if (MEMOIZE_OUTER_SCAN)
-        {
-            // Copy data back to smem
-            #pragma unroll
-            for (int i = 0; i < RAKING_SEGMENT; i++)
-            {
-                smem_raking_ptr[i] = cached_segment[i];
-            }
-        }
-    }
-
-
-    /**
-     * Reset shared memory digit counters
-     */
-    __device__ __forceinline__ void ResetCounters()
-    {
-        // Reset shared memory digit counters
-        #pragma unroll
-        for (int LANE = 0; LANE < PADDED_COUNTER_LANES; LANE++)
-        {
-            *((PackedCounter*) temp_storage.aliasable.digit_counters[LANE][linear_tid]) = 0;
-        }
-    }
-
-
-    /**
-     * Block-scan prefix callback
-     */
-    struct PrefixCallBack
-    {
-        __device__ __forceinline__ PackedCounter operator()(PackedCounter block_aggregate)
-        {
-            PackedCounter block_prefix = 0;
-
-            // Propagate totals in packed fields
-            #pragma unroll
-            for (int PACKED = 1; PACKED < PACKING_RATIO; PACKED++)
-            {
-                block_prefix += block_aggregate << (sizeof(DigitCounter) * 8 * PACKED);
-            }
-
-            return block_prefix;
-        }
-    };
-
-
-    /**
-     * Scan shared memory digit counters.
-     */
-    __device__ __forceinline__ void ScanCounters()
-    {
-        // Upsweep scan
-        PackedCounter raking_partial = Upsweep();
-
-        // Compute exclusive sum
-        PackedCounter exclusive_partial;
-        PrefixCallBack prefix_call_back;
-        BlockScan(temp_storage.block_scan).ExclusiveSum(raking_partial, exclusive_partial, prefix_call_back);
-
-        // Downsweep scan with exclusive partial
-        ExclusiveDownsweep(exclusive_partial);
-    }
-
-public:
-
-    /// \smemstorage{BlockScan}
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    /******************************************************************//**
-     * \name Collective constructors
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
-     */
-    __device__ __forceinline__ BlockRadixRank()
-    :
-        temp_storage(PrivateStorage()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    /**
-     * \brief Collective constructor using the specified memory allocation as temporary storage.
-     */
-    __device__ __forceinline__ BlockRadixRank(
-        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Raking
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Rank keys.
-     */
-    template <
-        typename        UnsignedBits,
-        int             KEYS_PER_THREAD>
-    __device__ __forceinline__ void RankKeys(
-        UnsignedBits    (&keys)[KEYS_PER_THREAD],           ///< [in] Keys for this tile
-        int             (&ranks)[KEYS_PER_THREAD],          ///< [out] For each key, the local rank within the tile
-        int             current_bit,                        ///< [in] The least-significant bit position of the current digit to extract
-        int             num_bits)                           ///< [in] The number of bits in the current digit
-    {
-        DigitCounter    thread_prefixes[KEYS_PER_THREAD];   // For each key, the count of previous keys in this tile having the same digit
-        DigitCounter*   digit_counters[KEYS_PER_THREAD];    // For each key, the byte-offset of its corresponding digit counter in smem
-
-        // Reset shared memory digit counters
-        ResetCounters();
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < KEYS_PER_THREAD; ++ITEM)
-        {
-            // Get digit
-            unsigned int digit = BFE(keys[ITEM], current_bit, num_bits);
-
-            // Get sub-counter
-            unsigned int sub_counter = digit >> LOG_COUNTER_LANES;
-
-            // Get counter lane
-            unsigned int counter_lane = digit & (COUNTER_LANES - 1);
-
-            if (IS_DESCENDING)
-            {
-                sub_counter = PACKING_RATIO - 1 - sub_counter;
-                counter_lane = COUNTER_LANES - 1 - counter_lane;
-            }
-
-            // Pointer to smem digit counter
-            digit_counters[ITEM] = &temp_storage.aliasable.digit_counters[counter_lane][linear_tid][sub_counter];
-
-            // Load thread-exclusive prefix
-            thread_prefixes[ITEM] = *digit_counters[ITEM];
-
-            // Store inclusive prefix
-            *digit_counters[ITEM] = thread_prefixes[ITEM] + 1;
-        }
-
-        CTA_SYNC();
-
-        // Scan shared memory counters
-        ScanCounters();
-
-        CTA_SYNC();
-
-        // Extract the local ranks of each key
-        for (int ITEM = 0; ITEM < KEYS_PER_THREAD; ++ITEM)
-        {
-            // Add in thread block exclusive prefix
-            ranks[ITEM] = thread_prefixes[ITEM] + *digit_counters[ITEM];
-        }
-    }
-
-
-    /**
-     * \brief Rank keys.  For the lower \p RADIX_DIGITS threads, digit counts for each digit are provided for the corresponding thread.
-     */
-    template <
-        typename        UnsignedBits,
-        int             KEYS_PER_THREAD>
-    __device__ __forceinline__ void RankKeys(
-        UnsignedBits    (&keys)[KEYS_PER_THREAD],           ///< [in] Keys for this tile
-        int             (&ranks)[KEYS_PER_THREAD],          ///< [out] For each key, the local rank within the tile (out parameter)
-        int             current_bit,                        ///< [in] The least-significant bit position of the current digit to extract
-        int             num_bits,                           ///< [in] The number of bits in the current digit
-        int             (&exclusive_digit_prefix)[BINS_TRACKED_PER_THREAD])            ///< [out] The exclusive prefix sum for the digits [(threadIdx.x * BINS_TRACKED_PER_THREAD) ... (threadIdx.x * BINS_TRACKED_PER_THREAD) + BINS_TRACKED_PER_THREAD - 1]
-    {
-        // Rank keys
-        RankKeys(keys, ranks, current_bit, num_bits);
-
-        // Get the inclusive and exclusive digit totals corresponding to the calling thread.
-        #pragma unroll
-        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
-        {
-            int bin_idx = (linear_tid * BINS_TRACKED_PER_THREAD) + track;
-
-            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
-            {
-                if (IS_DESCENDING)
-                    bin_idx = RADIX_DIGITS - bin_idx - 1;
-
-                // Obtain ex/inclusive digit counts.  (Unfortunately these all reside in the
-                // first counter column, resulting in unavoidable bank conflicts.)
-                unsigned int counter_lane   = (bin_idx & (COUNTER_LANES - 1));
-                unsigned int sub_counter    = bin_idx >> (LOG_COUNTER_LANES);
-
-                exclusive_digit_prefix[track] = temp_storage.aliasable.digit_counters[counter_lane][0][sub_counter];
-            }
-        }
-    }
-};
-
-
-
-
-
-/**
- * Radix-rank using match.any
- */
-template <
-    int                     BLOCK_DIM_X,
-    int                     RADIX_BITS,
-    bool                    IS_DESCENDING,
-    BlockScanAlgorithm      INNER_SCAN_ALGORITHM    = BLOCK_SCAN_WARP_SCANS,
-    int                     BLOCK_DIM_Y             = 1,
-    int                     BLOCK_DIM_Z             = 1,
-    int                     PTX_ARCH                = CUB_PTX_ARCH>
-class BlockRadixRankMatch
-{
-private:
-
-    /******************************************************************************
-     * Type definitions and constants
-     ******************************************************************************/
-
-    typedef int32_t    RankT;
-    typedef int32_t    DigitCounterT;
-
-    enum
-    {
-        // The thread block size in threads
-        BLOCK_THREADS               = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-
-        RADIX_DIGITS                = 1 << RADIX_BITS,
-
-        LOG_WARP_THREADS            = CUB_LOG_WARP_THREADS(PTX_ARCH),
-        WARP_THREADS                = 1 << LOG_WARP_THREADS,
-        WARPS                       = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS,
-
-        PADDED_WARPS            = ((WARPS & 0x1) == 0) ?
-                                    WARPS + 1 :
-                                    WARPS,
-
-        COUNTERS                = PADDED_WARPS * RADIX_DIGITS,
-        RAKING_SEGMENT          = (COUNTERS + BLOCK_THREADS - 1) / BLOCK_THREADS,
-        PADDED_RAKING_SEGMENT   = ((RAKING_SEGMENT & 0x1) == 0) ?
-                                    RAKING_SEGMENT + 1 :
-                                    RAKING_SEGMENT,
-    };
-
-public:
-
-    enum
-    {
-        /// Number of bin-starting offsets tracked per thread
-        BINS_TRACKED_PER_THREAD = CUB_MAX(1, (RADIX_DIGITS + BLOCK_THREADS - 1) / BLOCK_THREADS),
-    };
-
-private:
-
-    /// BlockScan type
-    typedef BlockScan<
-            DigitCounterT,
-            BLOCK_THREADS,
-            INNER_SCAN_ALGORITHM,
-            BLOCK_DIM_Y,
-            BLOCK_DIM_Z,
-            PTX_ARCH>
-        BlockScanT;
-
-
-    /// Shared memory storage layout type for BlockRadixRank
-    struct __align__(16) _TempStorage
-    {
-        typename BlockScanT::TempStorage            block_scan;
-
-        union __align__(16) Aliasable
-        {
-            volatile DigitCounterT                  warp_digit_counters[RADIX_DIGITS][PADDED_WARPS];
-            DigitCounterT                           raking_grid[BLOCK_THREADS][PADDED_RAKING_SEGMENT];
-
-        } aliasable;
-    };
-
-
-    /******************************************************************************
-     * Thread fields
-     ******************************************************************************/
-
-    /// Shared storage reference
-    _TempStorage &temp_storage;
-
-    /// Linear thread-id
-    unsigned int linear_tid;
-
-
-
-public:
-
-    /// \smemstorage{BlockScan}
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    /******************************************************************//**
-     * \name Collective constructors
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Collective constructor using the specified memory allocation as temporary storage.
-     */
-    __device__ __forceinline__ BlockRadixRankMatch(
-        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Raking
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Rank keys.
-     */
-    template <
-        typename        UnsignedBits,
-        int             KEYS_PER_THREAD>
-    __device__ __forceinline__ void RankKeys(
-        UnsignedBits    (&keys)[KEYS_PER_THREAD],           ///< [in] Keys for this tile
-        int             (&ranks)[KEYS_PER_THREAD],          ///< [out] For each key, the local rank within the tile
-        int             current_bit,                        ///< [in] The least-significant bit position of the current digit to extract
-        int             num_bits)                           ///< [in] The number of bits in the current digit
-    {
-        // Initialize shared digit counters
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < PADDED_RAKING_SEGMENT; ++ITEM)
-            temp_storage.aliasable.raking_grid[linear_tid][ITEM] = 0;
-
-        CTA_SYNC();
-
-        // Each warp will strip-mine its section of input, one strip at a time
-
-        volatile DigitCounterT  *digit_counters[KEYS_PER_THREAD];
-        uint32_t                warp_id         = linear_tid >> LOG_WARP_THREADS;
-        uint32_t                lane_mask_lt    = LaneMaskLt();
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < KEYS_PER_THREAD; ++ITEM)
-        {
-            // My digit
-            uint32_t digit = BFE(keys[ITEM], current_bit, num_bits);
-
-            if (IS_DESCENDING)
-                digit = RADIX_DIGITS - digit - 1;
-
-            // Mask of peers who have same digit as me
-            uint32_t peer_mask = MatchAny<RADIX_BITS>(digit);
-
-            // Pointer to smem digit counter for this key
-            digit_counters[ITEM] = &temp_storage.aliasable.warp_digit_counters[digit][warp_id];
-
-            // Number of occurrences in previous strips
-            DigitCounterT warp_digit_prefix = *digit_counters[ITEM];
-
-            // Warp-sync
-            WARP_SYNC(0xFFFFFFFF);
-
-            // Number of peers having same digit as me
-            int32_t digit_count = __popc(peer_mask);
-
-            // Number of lower-ranked peers having same digit seen so far
-            int32_t peer_digit_prefix = __popc(peer_mask & lane_mask_lt);
-
-            if (peer_digit_prefix == 0)
-            {
-                // First thread for each digit updates the shared warp counter
-                *digit_counters[ITEM] = DigitCounterT(warp_digit_prefix + digit_count);
-            }
-
-            // Warp-sync
-            WARP_SYNC(0xFFFFFFFF);
-
-            // Number of prior keys having same digit
-            ranks[ITEM] = warp_digit_prefix + DigitCounterT(peer_digit_prefix);
-        }
-
-        CTA_SYNC();
-
-        // Scan warp counters
-
-        DigitCounterT scan_counters[PADDED_RAKING_SEGMENT];
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < PADDED_RAKING_SEGMENT; ++ITEM)
-            scan_counters[ITEM] = temp_storage.aliasable.raking_grid[linear_tid][ITEM];
-
-        BlockScanT(temp_storage.block_scan).ExclusiveSum(scan_counters, scan_counters);
-
-        #pragma unroll
-        for (int ITEM = 0; ITEM < PADDED_RAKING_SEGMENT; ++ITEM)
-            temp_storage.aliasable.raking_grid[linear_tid][ITEM] = scan_counters[ITEM];
-
-        CTA_SYNC();
-
-        // Seed ranks with counter values from previous warps
-        #pragma unroll
-        for (int ITEM = 0; ITEM < KEYS_PER_THREAD; ++ITEM)
-            ranks[ITEM] += *digit_counters[ITEM];
-    }
-
-
-    /**
-     * \brief Rank keys.  For the lower \p RADIX_DIGITS threads, digit counts for each digit are provided for the corresponding thread.
-     */
-    template <
-        typename        UnsignedBits,
-        int             KEYS_PER_THREAD>
-    __device__ __forceinline__ void RankKeys(
-        UnsignedBits    (&keys)[KEYS_PER_THREAD],           ///< [in] Keys for this tile
-        int             (&ranks)[KEYS_PER_THREAD],          ///< [out] For each key, the local rank within the tile (out parameter)
-        int             current_bit,                        ///< [in] The least-significant bit position of the current digit to extract
-        int             num_bits,                           ///< [in] The number of bits in the current digit
-        int             (&exclusive_digit_prefix)[BINS_TRACKED_PER_THREAD])            ///< [out] The exclusive prefix sum for the digits [(threadIdx.x * BINS_TRACKED_PER_THREAD) ... (threadIdx.x * BINS_TRACKED_PER_THREAD) + BINS_TRACKED_PER_THREAD - 1]
-    {
-        RankKeys(keys, ranks, current_bit, num_bits);
-
-        // Get exclusive count for each digit
-        #pragma unroll
-        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
-        {
-            int bin_idx = (linear_tid * BINS_TRACKED_PER_THREAD) + track;
-
-            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
-            {
-                if (IS_DESCENDING)
-                    bin_idx = RADIX_DIGITS - bin_idx - 1;
-
-                exclusive_digit_prefix[track] = temp_storage.aliasable.warp_digit_counters[bin_idx][0];
-            }
-        }
-    }
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/block/block_radix_sort.cuh b/pfsimulator/third_party/cub/block/block_radix_sort.cuh
deleted file mode 100644
index ac0c9f85b..000000000
--- a/pfsimulator/third_party/cub/block/block_radix_sort.cuh
+++ /dev/null
@@ -1,863 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * The cub::BlockRadixSort class provides [<em>collective</em>](index.html#sec0) methods for radix sorting of items partitioned across a CUDA thread block.
- */
-
-
-#pragma once
-
-#include "block_exchange.cuh"
-#include "block_radix_rank.cuh"
-#include "../util_ptx.cuh"
-#include "../util_arch.cuh"
-#include "../util_type.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \brief The BlockRadixSort class provides [<em>collective</em>](index.html#sec0) methods for sorting items partitioned across a CUDA thread block using a radix sorting method.  ![](sorting_logo.png)
- * \ingroup BlockModule
- *
- * \tparam KeyT                 KeyT type
- * \tparam BLOCK_DIM_X          The thread block length in threads along the X dimension
- * \tparam ITEMS_PER_THREAD     The number of items per thread
- * \tparam ValueT               <b>[optional]</b> ValueT type (default: cub::NullType, which indicates a keys-only sort)
- * \tparam RADIX_BITS           <b>[optional]</b> The number of radix bits per digit place (default: 4 bits)
- * \tparam MEMOIZE_OUTER_SCAN   <b>[optional]</b> Whether or not to buffer outer raking scan partials to incur fewer shared memory reads at the expense of higher register pressure (default: true for architectures SM35 and newer, false otherwise).
- * \tparam INNER_SCAN_ALGORITHM <b>[optional]</b> The cub::BlockScanAlgorithm algorithm to use (default: cub::BLOCK_SCAN_WARP_SCANS)
- * \tparam SMEM_CONFIG          <b>[optional]</b> Shared memory bank mode (default: \p cudaSharedMemBankSizeFourByte)
- * \tparam BLOCK_DIM_Y          <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
- * \tparam BLOCK_DIM_Z          <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
- * \tparam PTX_ARCH             <b>[optional]</b> \ptxversion
- *
- * \par Overview
- * - The [<em>radix sorting method</em>](http://en.wikipedia.org/wiki/Radix_sort) arranges
- *   items into ascending order.  It relies upon a positional representation for
- *   keys, i.e., each key is comprised of an ordered sequence of symbols (e.g., digits,
- *   characters, etc.) specified from least-significant to most-significant.  For a
- *   given input sequence of keys and a set of rules specifying a total ordering
- *   of the symbolic alphabet, the radix sorting method produces a lexicographic
- *   ordering of those keys.
- * - BlockRadixSort can sort all of the built-in C++ numeric primitive types
- *   (<tt>unsigned char</tt>, \p int, \p double, etc.) as well as CUDA's \p __half
- *   half-precision floating-point type. Within each key, the implementation treats fixed-length
- *   bit-sequences of \p RADIX_BITS as radix digit places.  Although the direct radix sorting
- *   method can only be applied to unsigned integral types, BlockRadixSort
- *   is able to sort signed and floating-point types via simple bit-wise transformations
- *   that ensure lexicographic key ordering.
- * - \rowmajor
- *
- * \par Performance Considerations
- * - \granularity
- *
- * \par A Simple Example
- * \blockcollective{BlockRadixSort}
- * \par
- * The code snippet below illustrates a sort of 512 integer keys that
- * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
- * where each thread owns 4 consecutive items.
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/block/block_radix_sort.cuh>
- *
- * __global__ void ExampleKernel(...)
- * {
- *     // Specialize BlockRadixSort for a 1D block of 128 threads owning 4 integer items each
- *     typedef cub::BlockRadixSort<int, 128, 4> BlockRadixSort;
- *
- *     // Allocate shared memory for BlockRadixSort
- *     __shared__ typename BlockRadixSort::TempStorage temp_storage;
- *
- *     // Obtain a segment of consecutive items that are blocked across threads
- *     int thread_keys[4];
- *     ...
- *
- *     // Collectively sort the keys
- *     BlockRadixSort(temp_storage).Sort(thread_keys);
- *
- *     ...
- * \endcode
- * \par
- * Suppose the set of input \p thread_keys across the block of threads is
- * <tt>{ [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }</tt>.  The
- * corresponding output \p thread_keys in those threads will be
- * <tt>{ [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }</tt>.
- *
- */
-template <
-    typename                KeyT,
-    int                     BLOCK_DIM_X,
-    int                     ITEMS_PER_THREAD,
-    typename                ValueT                   = NullType,
-    int                     RADIX_BITS              = 4,
-    bool                    MEMOIZE_OUTER_SCAN      = (CUB_PTX_ARCH >= 350) ? true : false,
-    BlockScanAlgorithm      INNER_SCAN_ALGORITHM    = BLOCK_SCAN_WARP_SCANS,
-    cudaSharedMemConfig     SMEM_CONFIG             = cudaSharedMemBankSizeFourByte,
-    int                     BLOCK_DIM_Y             = 1,
-    int                     BLOCK_DIM_Z             = 1,
-    int                     PTX_ARCH                = CUB_PTX_ARCH>
-class BlockRadixSort
-{
-private:
-
-    /******************************************************************************
-     * Constants and type definitions
-     ******************************************************************************/
-
-    enum
-    {
-        // The thread block size in threads
-        BLOCK_THREADS               = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-
-        // Whether or not there are values to be trucked along with keys
-        KEYS_ONLY                   = Equals<ValueT, NullType>::VALUE,
-    };
-
-    // KeyT traits and unsigned bits type
-    typedef Traits<KeyT>                        KeyTraits;
-    typedef typename KeyTraits::UnsignedBits    UnsignedBits;
-
-    /// Ascending BlockRadixRank utility type
-    typedef BlockRadixRank<
-            BLOCK_DIM_X,
-            RADIX_BITS,
-            false,
-            MEMOIZE_OUTER_SCAN,
-            INNER_SCAN_ALGORITHM,
-            SMEM_CONFIG,
-            BLOCK_DIM_Y,
-            BLOCK_DIM_Z,
-            PTX_ARCH>
-        AscendingBlockRadixRank;
-
-    /// Descending BlockRadixRank utility type
-    typedef BlockRadixRank<
-            BLOCK_DIM_X,
-            RADIX_BITS,
-            true,
-            MEMOIZE_OUTER_SCAN,
-            INNER_SCAN_ALGORITHM,
-            SMEM_CONFIG,
-            BLOCK_DIM_Y,
-            BLOCK_DIM_Z,
-            PTX_ARCH>
-        DescendingBlockRadixRank;
-
-    /// BlockExchange utility type for keys
-    typedef BlockExchange<KeyT, BLOCK_DIM_X, ITEMS_PER_THREAD, false, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> BlockExchangeKeys;
-
-    /// BlockExchange utility type for values
-    typedef BlockExchange<ValueT, BLOCK_DIM_X, ITEMS_PER_THREAD, false, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> BlockExchangeValues;
-
-    /// Shared memory storage layout type
-    union _TempStorage
-    {
-        typename AscendingBlockRadixRank::TempStorage  asending_ranking_storage;
-        typename DescendingBlockRadixRank::TempStorage descending_ranking_storage;
-        typename BlockExchangeKeys::TempStorage        exchange_keys;
-        typename BlockExchangeValues::TempStorage      exchange_values;
-    };
-
-
-    /******************************************************************************
-     * Thread fields
-     ******************************************************************************/
-
-    /// Shared storage reference
-    _TempStorage &temp_storage;
-
-    /// Linear thread-id
-    unsigned int linear_tid;
-
-    /******************************************************************************
-     * Utility methods
-     ******************************************************************************/
-
-    /// Internal storage allocator
-    __device__ __forceinline__ _TempStorage& PrivateStorage()
-    {
-        __shared__ _TempStorage private_storage;
-        return private_storage;
-    }
-
-    /// Rank keys (specialized for ascending sort)
-    __device__ __forceinline__ void RankKeys(
-        UnsignedBits    (&unsigned_keys)[ITEMS_PER_THREAD],
-        int             (&ranks)[ITEMS_PER_THREAD],
-        int             begin_bit,
-        int             pass_bits,
-        Int2Type<false> /*is_descending*/)
-    {
-        AscendingBlockRadixRank(temp_storage.asending_ranking_storage).RankKeys(
-            unsigned_keys,
-            ranks,
-            begin_bit,
-            pass_bits);
-    }
-
-    /// Rank keys (specialized for descending sort)
-    __device__ __forceinline__ void RankKeys(
-        UnsignedBits    (&unsigned_keys)[ITEMS_PER_THREAD],
-        int             (&ranks)[ITEMS_PER_THREAD],
-        int             begin_bit,
-        int             pass_bits,
-        Int2Type<true>  /*is_descending*/)
-    {
-        DescendingBlockRadixRank(temp_storage.descending_ranking_storage).RankKeys(
-            unsigned_keys,
-            ranks,
-            begin_bit,
-            pass_bits);
-    }
-
-    /// ExchangeValues (specialized for key-value sort, to-blocked arrangement)
-    __device__ __forceinline__ void ExchangeValues(
-        ValueT          (&values)[ITEMS_PER_THREAD],
-        int             (&ranks)[ITEMS_PER_THREAD],
-        Int2Type<false> /*is_keys_only*/,
-        Int2Type<true>  /*is_blocked*/)
-    {
-        CTA_SYNC();
-
-        // Exchange values through shared memory in blocked arrangement
-        BlockExchangeValues(temp_storage.exchange_values).ScatterToBlocked(values, ranks);
-    }
-
-    /// ExchangeValues (specialized for key-value sort, to-striped arrangement)
-    __device__ __forceinline__ void ExchangeValues(
-        ValueT          (&values)[ITEMS_PER_THREAD],
-        int             (&ranks)[ITEMS_PER_THREAD],
-        Int2Type<false> /*is_keys_only*/,
-        Int2Type<false> /*is_blocked*/)
-    {
-        CTA_SYNC();
-
-        // Exchange values through shared memory in blocked arrangement
-        BlockExchangeValues(temp_storage.exchange_values).ScatterToStriped(values, ranks);
-    }
-
-    /// ExchangeValues (specialized for keys-only sort)
-    template <int IS_BLOCKED>
-    __device__ __forceinline__ void ExchangeValues(
-        ValueT                  (&/*values*/)[ITEMS_PER_THREAD],
-        int                     (&/*ranks*/)[ITEMS_PER_THREAD],
-        Int2Type<true>          /*is_keys_only*/,
-        Int2Type<IS_BLOCKED>    /*is_blocked*/)
-    {}
-
-    /// Sort blocked arrangement
-    template <int DESCENDING, int KEYS_ONLY>
-    __device__ __forceinline__ void SortBlocked(
-        KeyT                    (&keys)[ITEMS_PER_THREAD],          ///< Keys to sort
-        ValueT                  (&values)[ITEMS_PER_THREAD],        ///< Values to sort
-        int                     begin_bit,                          ///< The beginning (least-significant) bit index needed for key comparison
-        int                     end_bit,                            ///< The past-the-end (most-significant) bit index needed for key comparison
-        Int2Type<DESCENDING>    is_descending,                      ///< Tag whether is a descending-order sort
-        Int2Type<KEYS_ONLY>     is_keys_only)                       ///< Tag whether is keys-only sort
-    {
-        UnsignedBits (&unsigned_keys)[ITEMS_PER_THREAD] =
-            reinterpret_cast<UnsignedBits (&)[ITEMS_PER_THREAD]>(keys);
-
-        // Twiddle bits if necessary
-        #pragma unroll
-        for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++)
-        {
-            unsigned_keys[KEY] = KeyTraits::TwiddleIn(unsigned_keys[KEY]);
-        }
-
-        // Radix sorting passes
-        while (true)
-        {
-            int pass_bits = CUB_MIN(RADIX_BITS, end_bit - begin_bit);
-
-            // Rank the blocked keys
-            int ranks[ITEMS_PER_THREAD];
-            RankKeys(unsigned_keys, ranks, begin_bit, pass_bits, is_descending);
-            begin_bit += RADIX_BITS;
-
-            CTA_SYNC();
-
-            // Exchange keys through shared memory in blocked arrangement
-            BlockExchangeKeys(temp_storage.exchange_keys).ScatterToBlocked(keys, ranks);
-
-            // Exchange values through shared memory in blocked arrangement
-            ExchangeValues(values, ranks, is_keys_only, Int2Type<true>());
-
-            // Quit if done
-            if (begin_bit >= end_bit) break;
-
-            CTA_SYNC();
-        }
-
-        // Untwiddle bits if necessary
-        #pragma unroll
-        for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++)
-        {
-            unsigned_keys[KEY] = KeyTraits::TwiddleOut(unsigned_keys[KEY]);
-        }
-    }
-
-public:
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-    /// Sort blocked -> striped arrangement
-    template <int DESCENDING, int KEYS_ONLY>
-    __device__ __forceinline__ void SortBlockedToStriped(
-        KeyT                    (&keys)[ITEMS_PER_THREAD],          ///< Keys to sort
-        ValueT                  (&values)[ITEMS_PER_THREAD],        ///< Values to sort
-        int                     begin_bit,                          ///< The beginning (least-significant) bit index needed for key comparison
-        int                     end_bit,                            ///< The past-the-end (most-significant) bit index needed for key comparison
-        Int2Type<DESCENDING>    is_descending,                      ///< Tag whether is a descending-order sort
-        Int2Type<KEYS_ONLY>     is_keys_only)                       ///< Tag whether is keys-only sort
-    {
-        UnsignedBits (&unsigned_keys)[ITEMS_PER_THREAD] =
-            reinterpret_cast<UnsignedBits (&)[ITEMS_PER_THREAD]>(keys);
-
-        // Twiddle bits if necessary
-        #pragma unroll
-        for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++)
-        {
-            unsigned_keys[KEY] = KeyTraits::TwiddleIn(unsigned_keys[KEY]);
-        }
-
-        // Radix sorting passes
-        while (true)
-        {
-            int pass_bits = CUB_MIN(RADIX_BITS, end_bit - begin_bit);
-
-            // Rank the blocked keys
-            int ranks[ITEMS_PER_THREAD];
-            RankKeys(unsigned_keys, ranks, begin_bit, pass_bits, is_descending);
-            begin_bit += RADIX_BITS;
-
-            CTA_SYNC();
-
-            // Check if this is the last pass
-            if (begin_bit >= end_bit)
-            {
-                // Last pass exchanges keys through shared memory in striped arrangement
-                BlockExchangeKeys(temp_storage.exchange_keys).ScatterToStriped(keys, ranks);
-
-                // Last pass exchanges through shared memory in striped arrangement
-                ExchangeValues(values, ranks, is_keys_only, Int2Type<false>());
-
-                // Quit
-                break;
-            }
-
-            // Exchange keys through shared memory in blocked arrangement
-            BlockExchangeKeys(temp_storage.exchange_keys).ScatterToBlocked(keys, ranks);
-
-            // Exchange values through shared memory in blocked arrangement
-            ExchangeValues(values, ranks, is_keys_only, Int2Type<true>());
-
-            CTA_SYNC();
-        }
-
-        // Untwiddle bits if necessary
-        #pragma unroll
-        for (int KEY = 0; KEY < ITEMS_PER_THREAD; KEY++)
-        {
-            unsigned_keys[KEY] = KeyTraits::TwiddleOut(unsigned_keys[KEY]);
-        }
-    }
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-    /// \smemstorage{BlockRadixSort}
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    /******************************************************************//**
-     * \name Collective constructors
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
-     */
-    __device__ __forceinline__ BlockRadixSort()
-    :
-        temp_storage(PrivateStorage()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    /**
-     * \brief Collective constructor using the specified memory allocation as temporary storage.
-     */
-    __device__ __forceinline__ BlockRadixSort(
-        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Sorting (blocked arrangements)
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Performs an ascending block-wide radix sort over a [<em>blocked arrangement</em>](index.html#sec5sec3) of keys.
-     *
-     * \par
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a sort of 512 integer keys that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive keys.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_radix_sort.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockRadixSort for a 1D block of 128 threads owning 4 integer keys each
-     *     typedef cub::BlockRadixSort<int, 128, 4> BlockRadixSort;
-     *
-     *     // Allocate shared memory for BlockRadixSort
-     *     __shared__ typename BlockRadixSort::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_keys[4];
-     *     ...
-     *
-     *     // Collectively sort the keys
-     *     BlockRadixSort(temp_storage).Sort(thread_keys);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_keys across the block of threads is
-     * <tt>{ [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }</tt>.
-     * The corresponding output \p thread_keys in those threads will be
-     * <tt>{ [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }</tt>.
-     */
-    __device__ __forceinline__ void Sort(
-        KeyT    (&keys)[ITEMS_PER_THREAD],          ///< [in-out] Keys to sort
-        int     begin_bit   = 0,                    ///< [in] <b>[optional]</b> The beginning (least-significant) bit index needed for key comparison
-        int     end_bit     = sizeof(KeyT) * 8)      ///< [in] <b>[optional]</b> The past-the-end (most-significant) bit index needed for key comparison
-    {
-        NullType values[ITEMS_PER_THREAD];
-
-        SortBlocked(keys, values, begin_bit, end_bit, Int2Type<false>(), Int2Type<KEYS_ONLY>());
-    }
-
-
-    /**
-     * \brief Performs an ascending block-wide radix sort across a [<em>blocked arrangement</em>](index.html#sec5sec3) of keys and values.
-     *
-     * \par
-     * - BlockRadixSort can only accommodate one associated tile of values. To "truck along"
-     *   more than one tile of values, simply perform a key-value sort of the keys paired
-     *   with a temporary value array that enumerates the key indices.  The reordered indices
-     *   can then be used as a gather-vector for exchanging other associated tile data through
-     *   shared memory.
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a sort of 512 integer keys and values that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive pairs.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_radix_sort.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockRadixSort for a 1D block of 128 threads owning 4 integer keys and values each
-     *     typedef cub::BlockRadixSort<int, 128, 4, int> BlockRadixSort;
-     *
-     *     // Allocate shared memory for BlockRadixSort
-     *     __shared__ typename BlockRadixSort::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_keys[4];
-     *     int thread_values[4];
-     *     ...
-     *
-     *     // Collectively sort the keys and values among block threads
-     *     BlockRadixSort(temp_storage).Sort(thread_keys, thread_values);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_keys across the block of threads is
-     * <tt>{ [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }</tt>.  The
-     * corresponding output \p thread_keys in those threads will be
-     * <tt>{ [0,1,2,3], [4,5,6,7], [8,9,10,11], ..., [508,509,510,511] }</tt>.
-     *
-     */
-    __device__ __forceinline__ void Sort(
-        KeyT    (&keys)[ITEMS_PER_THREAD],          ///< [in-out] Keys to sort
-        ValueT  (&values)[ITEMS_PER_THREAD],        ///< [in-out] Values to sort
-        int     begin_bit   = 0,                    ///< [in] <b>[optional]</b> The beginning (least-significant) bit index needed for key comparison
-        int     end_bit     = sizeof(KeyT) * 8)      ///< [in] <b>[optional]</b> The past-the-end (most-significant) bit index needed for key comparison
-    {
-        SortBlocked(keys, values, begin_bit, end_bit, Int2Type<false>(), Int2Type<KEYS_ONLY>());
-    }
-
-    /**
-     * \brief Performs a descending block-wide radix sort over a [<em>blocked arrangement</em>](index.html#sec5sec3) of keys.
-     *
-     * \par
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a sort of 512 integer keys that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive keys.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_radix_sort.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockRadixSort for a 1D block of 128 threads owning 4 integer keys each
-     *     typedef cub::BlockRadixSort<int, 128, 4> BlockRadixSort;
-     *
-     *     // Allocate shared memory for BlockRadixSort
-     *     __shared__ typename BlockRadixSort::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_keys[4];
-     *     ...
-     *
-     *     // Collectively sort the keys
-     *     BlockRadixSort(temp_storage).Sort(thread_keys);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_keys across the block of threads is
-     * <tt>{ [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }</tt>.
-     * The corresponding output \p thread_keys in those threads will be
-     * <tt>{ [511,510,509,508], [11,10,9,8], [7,6,5,4], ..., [3,2,1,0] }</tt>.
-     */
-    __device__ __forceinline__ void SortDescending(
-        KeyT    (&keys)[ITEMS_PER_THREAD],          ///< [in-out] Keys to sort
-        int     begin_bit   = 0,                    ///< [in] <b>[optional]</b> The beginning (least-significant) bit index needed for key comparison
-        int     end_bit     = sizeof(KeyT) * 8)      ///< [in] <b>[optional]</b> The past-the-end (most-significant) bit index needed for key comparison
-    {
-        NullType values[ITEMS_PER_THREAD];
-
-        SortBlocked(keys, values, begin_bit, end_bit, Int2Type<true>(), Int2Type<KEYS_ONLY>());
-    }
-
-
-    /**
-     * \brief Performs a descending block-wide radix sort across a [<em>blocked arrangement</em>](index.html#sec5sec3) of keys and values.
-     *
-     * \par
-     * - BlockRadixSort can only accommodate one associated tile of values. To "truck along"
-     *   more than one tile of values, simply perform a key-value sort of the keys paired
-     *   with a temporary value array that enumerates the key indices.  The reordered indices
-     *   can then be used as a gather-vector for exchanging other associated tile data through
-     *   shared memory.
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a sort of 512 integer keys and values that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive pairs.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_radix_sort.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockRadixSort for a 1D block of 128 threads owning 4 integer keys and values each
-     *     typedef cub::BlockRadixSort<int, 128, 4, int> BlockRadixSort;
-     *
-     *     // Allocate shared memory for BlockRadixSort
-     *     __shared__ typename BlockRadixSort::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_keys[4];
-     *     int thread_values[4];
-     *     ...
-     *
-     *     // Collectively sort the keys and values among block threads
-     *     BlockRadixSort(temp_storage).Sort(thread_keys, thread_values);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_keys across the block of threads is
-     * <tt>{ [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }</tt>.  The
-     * corresponding output \p thread_keys in those threads will be
-     * <tt>{ [511,510,509,508], [11,10,9,8], [7,6,5,4], ..., [3,2,1,0] }</tt>.
-     *
-     */
-    __device__ __forceinline__ void SortDescending(
-        KeyT    (&keys)[ITEMS_PER_THREAD],          ///< [in-out] Keys to sort
-        ValueT  (&values)[ITEMS_PER_THREAD],        ///< [in-out] Values to sort
-        int     begin_bit   = 0,                    ///< [in] <b>[optional]</b> The beginning (least-significant) bit index needed for key comparison
-        int     end_bit     = sizeof(KeyT) * 8)      ///< [in] <b>[optional]</b> The past-the-end (most-significant) bit index needed for key comparison
-    {
-        SortBlocked(keys, values, begin_bit, end_bit, Int2Type<true>(), Int2Type<KEYS_ONLY>());
-    }
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Sorting (blocked arrangement -> striped arrangement)
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Performs an ascending radix sort across a [<em>blocked arrangement</em>](index.html#sec5sec3) of keys, leaving them in a [<em>striped arrangement</em>](index.html#sec5sec3).
-     *
-     * \par
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a sort of 512 integer keys that
-     * are initially partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive keys.  The final partitioning is striped.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_radix_sort.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockRadixSort for a 1D block of 128 threads owning 4 integer keys each
-     *     typedef cub::BlockRadixSort<int, 128, 4> BlockRadixSort;
-     *
-     *     // Allocate shared memory for BlockRadixSort
-     *     __shared__ typename BlockRadixSort::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_keys[4];
-     *     ...
-     *
-     *     // Collectively sort the keys
-     *     BlockRadixSort(temp_storage).SortBlockedToStriped(thread_keys);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_keys across the block of threads is
-     * <tt>{ [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }</tt>.  The
-     * corresponding output \p thread_keys in those threads will be
-     * <tt>{ [0,128,256,384], [1,129,257,385], [2,130,258,386], ..., [127,255,383,511] }</tt>.
-     *
-     */
-    __device__ __forceinline__ void SortBlockedToStriped(
-        KeyT    (&keys)[ITEMS_PER_THREAD],          ///< [in-out] Keys to sort
-        int     begin_bit   = 0,                    ///< [in] <b>[optional]</b> The beginning (least-significant) bit index needed for key comparison
-        int     end_bit     = sizeof(KeyT) * 8)      ///< [in] <b>[optional]</b> The past-the-end (most-significant) bit index needed for key comparison
-    {
-        NullType values[ITEMS_PER_THREAD];
-
-        SortBlockedToStriped(keys, values, begin_bit, end_bit, Int2Type<false>(), Int2Type<KEYS_ONLY>());
-    }
-
-
-    /**
-     * \brief Performs an ascending radix sort across a [<em>blocked arrangement</em>](index.html#sec5sec3) of keys and values, leaving them in a [<em>striped arrangement</em>](index.html#sec5sec3).
-     *
-     * \par
-     * - BlockRadixSort can only accommodate one associated tile of values. To "truck along"
-     *   more than one tile of values, simply perform a key-value sort of the keys paired
-     *   with a temporary value array that enumerates the key indices.  The reordered indices
-     *   can then be used as a gather-vector for exchanging other associated tile data through
-     *   shared memory.
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a sort of 512 integer keys and values that
-     * are initially partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive pairs.  The final partitioning is striped.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_radix_sort.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockRadixSort for a 1D block of 128 threads owning 4 integer keys and values each
-     *     typedef cub::BlockRadixSort<int, 128, 4, int> BlockRadixSort;
-     *
-     *     // Allocate shared memory for BlockRadixSort
-     *     __shared__ typename BlockRadixSort::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_keys[4];
-     *     int thread_values[4];
-     *     ...
-     *
-     *     // Collectively sort the keys and values among block threads
-     *     BlockRadixSort(temp_storage).SortBlockedToStriped(thread_keys, thread_values);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_keys across the block of threads is
-     * <tt>{ [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }</tt>.  The
-     * corresponding output \p thread_keys in those threads will be
-     * <tt>{ [0,128,256,384], [1,129,257,385], [2,130,258,386], ..., [127,255,383,511] }</tt>.
-     *
-     */
-    __device__ __forceinline__ void SortBlockedToStriped(
-        KeyT    (&keys)[ITEMS_PER_THREAD],          ///< [in-out] Keys to sort
-        ValueT  (&values)[ITEMS_PER_THREAD],        ///< [in-out] Values to sort
-        int     begin_bit   = 0,                    ///< [in] <b>[optional]</b> The beginning (least-significant) bit index needed for key comparison
-        int     end_bit     = sizeof(KeyT) * 8)      ///< [in] <b>[optional]</b> The past-the-end (most-significant) bit index needed for key comparison
-    {
-        SortBlockedToStriped(keys, values, begin_bit, end_bit, Int2Type<false>(), Int2Type<KEYS_ONLY>());
-    }
-
-
-    /**
-     * \brief Performs a descending radix sort across a [<em>blocked arrangement</em>](index.html#sec5sec3) of keys, leaving them in a [<em>striped arrangement</em>](index.html#sec5sec3).
-     *
-     * \par
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a sort of 512 integer keys that
-     * are initially partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive keys.  The final partitioning is striped.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_radix_sort.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockRadixSort for a 1D block of 128 threads owning 4 integer keys each
-     *     typedef cub::BlockRadixSort<int, 128, 4> BlockRadixSort;
-     *
-     *     // Allocate shared memory for BlockRadixSort
-     *     __shared__ typename BlockRadixSort::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_keys[4];
-     *     ...
-     *
-     *     // Collectively sort the keys
-     *     BlockRadixSort(temp_storage).SortBlockedToStriped(thread_keys);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_keys across the block of threads is
-     * <tt>{ [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }</tt>.  The
-     * corresponding output \p thread_keys in those threads will be
-     * <tt>{ [511,383,255,127], [386,258,130,2], [385,257,128,1], ..., [384,256,128,0] }</tt>.
-     *
-     */
-    __device__ __forceinline__ void SortDescendingBlockedToStriped(
-        KeyT    (&keys)[ITEMS_PER_THREAD],          ///< [in-out] Keys to sort
-        int     begin_bit   = 0,                    ///< [in] <b>[optional]</b> The beginning (least-significant) bit index needed for key comparison
-        int     end_bit     = sizeof(KeyT) * 8)      ///< [in] <b>[optional]</b> The past-the-end (most-significant) bit index needed for key comparison
-    {
-        NullType values[ITEMS_PER_THREAD];
-
-        SortBlockedToStriped(keys, values, begin_bit, end_bit, Int2Type<true>(), Int2Type<KEYS_ONLY>());
-    }
-
-
-    /**
-     * \brief Performs a descending radix sort across a [<em>blocked arrangement</em>](index.html#sec5sec3) of keys and values, leaving them in a [<em>striped arrangement</em>](index.html#sec5sec3).
-     *
-     * \par
-     * - BlockRadixSort can only accommodate one associated tile of values. To "truck along"
-     *   more than one tile of values, simply perform a key-value sort of the keys paired
-     *   with a temporary value array that enumerates the key indices.  The reordered indices
-     *   can then be used as a gather-vector for exchanging other associated tile data through
-     *   shared memory.
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a sort of 512 integer keys and values that
-     * are initially partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive pairs.  The final partitioning is striped.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_radix_sort.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockRadixSort for a 1D block of 128 threads owning 4 integer keys and values each
-     *     typedef cub::BlockRadixSort<int, 128, 4, int> BlockRadixSort;
-     *
-     *     // Allocate shared memory for BlockRadixSort
-     *     __shared__ typename BlockRadixSort::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_keys[4];
-     *     int thread_values[4];
-     *     ...
-     *
-     *     // Collectively sort the keys and values among block threads
-     *     BlockRadixSort(temp_storage).SortBlockedToStriped(thread_keys, thread_values);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_keys across the block of threads is
-     * <tt>{ [0,511,1,510], [2,509,3,508], [4,507,5,506], ..., [254,257,255,256] }</tt>.  The
-     * corresponding output \p thread_keys in those threads will be
-     * <tt>{ [511,383,255,127], [386,258,130,2], [385,257,128,1], ..., [384,256,128,0] }</tt>.
-     *
-     */
-    __device__ __forceinline__ void SortDescendingBlockedToStriped(
-        KeyT    (&keys)[ITEMS_PER_THREAD],          ///< [in-out] Keys to sort
-        ValueT  (&values)[ITEMS_PER_THREAD],        ///< [in-out] Values to sort
-        int     begin_bit   = 0,                    ///< [in] <b>[optional]</b> The beginning (least-significant) bit index needed for key comparison
-        int     end_bit     = sizeof(KeyT) * 8)      ///< [in] <b>[optional]</b> The past-the-end (most-significant) bit index needed for key comparison
-    {
-        SortBlockedToStriped(keys, values, begin_bit, end_bit, Int2Type<true>(), Int2Type<KEYS_ONLY>());
-    }
-
-
-    //@}  end member group
-
-};
-
-/**
- * \example example_block_radix_sort.cu
- */
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/block/block_raking_layout.cuh b/pfsimulator/third_party/cub/block/block_raking_layout.cuh
deleted file mode 100644
index 350061686..000000000
--- a/pfsimulator/third_party/cub/block/block_raking_layout.cuh
+++ /dev/null
@@ -1,152 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::BlockRakingLayout provides a conflict-free shared memory layout abstraction for warp-raking across thread block data.
- */
-
-
-#pragma once
-
-#include "../util_macro.cuh"
-#include "../util_arch.cuh"
-#include "../util_type.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \brief BlockRakingLayout provides a conflict-free shared memory layout abstraction for 1D raking across thread block data.    ![](raking.png)
- * \ingroup BlockModule
- *
- * \par Overview
- * This type facilitates a shared memory usage pattern where a block of CUDA
- * threads places elements into shared memory and then reduces the active
- * parallelism to one "raking" warp of threads for serially aggregating consecutive
- * sequences of shared items.  Padding is inserted to eliminate bank conflicts
- * (for most data types).
- *
- * \tparam T                        The data type to be exchanged.
- * \tparam BLOCK_THREADS            The thread block size in threads.
- * \tparam PTX_ARCH                 <b>[optional]</b> \ptxversion
- */
-template <
-    typename    T,
-    int         BLOCK_THREADS,
-    int         PTX_ARCH = CUB_PTX_ARCH>
-struct BlockRakingLayout
-{
-    //---------------------------------------------------------------------
-    // Constants and type definitions
-    //---------------------------------------------------------------------
-
-    enum
-    {
-        /// The total number of elements that need to be cooperatively reduced
-        SHARED_ELEMENTS = BLOCK_THREADS,
-
-        /// Maximum number of warp-synchronous raking threads
-        MAX_RAKING_THREADS = CUB_MIN(BLOCK_THREADS, CUB_WARP_THREADS(PTX_ARCH)),
-
-        /// Number of raking elements per warp-synchronous raking thread (rounded up)
-        SEGMENT_LENGTH = (SHARED_ELEMENTS + MAX_RAKING_THREADS - 1) / MAX_RAKING_THREADS,
-
-        /// Never use a raking thread that will have no valid data (e.g., when BLOCK_THREADS is 62 and SEGMENT_LENGTH is 2, we should only use 31 raking threads)
-        RAKING_THREADS = (SHARED_ELEMENTS + SEGMENT_LENGTH - 1) / SEGMENT_LENGTH,
-
-        /// Whether we will have bank conflicts (technically we should find out if the GCD is > 1)
-        HAS_CONFLICTS = (CUB_SMEM_BANKS(PTX_ARCH) % SEGMENT_LENGTH == 0),
-
-        /// Degree of bank conflicts (e.g., 4-way)
-        CONFLICT_DEGREE = (HAS_CONFLICTS) ?
-            (MAX_RAKING_THREADS * SEGMENT_LENGTH) / CUB_SMEM_BANKS(PTX_ARCH) :
-            1,
-
-        /// Pad each segment length with one element if segment length is not relatively prime to warp size and can't be optimized as a vector load
-        USE_SEGMENT_PADDING = ((SEGMENT_LENGTH & 1) == 0) && (SEGMENT_LENGTH > 2),
-
-        /// Total number of elements in the raking grid
-        GRID_ELEMENTS = RAKING_THREADS * (SEGMENT_LENGTH + USE_SEGMENT_PADDING),
-
-        /// Whether or not we need bounds checking during raking (the number of reduction elements is not a multiple of the number of raking threads)
-        UNGUARDED = (SHARED_ELEMENTS % RAKING_THREADS == 0),
-    };
-
-
-    /**
-     * \brief Shared memory storage type
-     */
-    struct __align__(16) _TempStorage
-    {
-        T buff[BlockRakingLayout::GRID_ELEMENTS];
-    };
-
-    /// Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    /**
-     * \brief Returns the location for the calling thread to place data into the grid
-     */
-    static __device__ __forceinline__ T* PlacementPtr(
-        TempStorage &temp_storage,
-        unsigned int linear_tid)
-    {
-        // Offset for partial
-        unsigned int offset = linear_tid;
-
-        // Add in one padding element for every segment
-        if (USE_SEGMENT_PADDING > 0)
-        {
-            offset += offset / SEGMENT_LENGTH;
-        }
-
-        // Incorporating a block of padding partials every shared memory segment
-        return temp_storage.Alias().buff + offset;
-    }
-
-
-    /**
-     * \brief Returns the location for the calling thread to begin sequential raking
-     */
-    static __device__ __forceinline__ T* RakingPtr(
-        TempStorage &temp_storage,
-        unsigned int linear_tid)
-    {
-        return temp_storage.Alias().buff + (linear_tid * (SEGMENT_LENGTH + USE_SEGMENT_PADDING));
-    }
-};
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/block/block_reduce.cuh b/pfsimulator/third_party/cub/block/block_reduce.cuh
deleted file mode 100644
index 261f2ea6f..000000000
--- a/pfsimulator/third_party/cub/block/block_reduce.cuh
+++ /dev/null
@@ -1,607 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * The cub::BlockReduce class provides [<em>collective</em>](index.html#sec0) methods for computing a parallel reduction of items partitioned across a CUDA thread block.
- */
-
-#pragma once
-
-#include "specializations/block_reduce_raking.cuh"
-#include "specializations/block_reduce_raking_commutative_only.cuh"
-#include "specializations/block_reduce_warp_reductions.cuh"
-#include "../util_ptx.cuh"
-#include "../util_type.cuh"
-#include "../thread/thread_operators.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-
-/******************************************************************************
- * Algorithmic variants
- ******************************************************************************/
-
-/**
- * BlockReduceAlgorithm enumerates alternative algorithms for parallel
- * reduction across a CUDA thread block.
- */
-enum BlockReduceAlgorithm
-{
-
-    /**
-     * \par Overview
-     * An efficient "raking" reduction algorithm that only supports commutative
-     * reduction operators (true for most operations, e.g., addition).
-     *
-     * \par
-     * Execution is comprised of three phases:
-     * -# Upsweep sequential reduction in registers (if threads contribute more
-     *    than one input each).  Threads in warps other than the first warp place
-     *    their partial reductions into shared memory.
-     * -# Upsweep sequential reduction in shared memory.  Threads within the first
-     *    warp continue to accumulate by raking across segments of shared partial reductions
-     * -# A warp-synchronous Kogge-Stone style reduction within the raking warp.
-     *
-     * \par
-     * \image html block_reduce.png
-     * <div class="centercaption">\p BLOCK_REDUCE_RAKING data flow for a hypothetical 16-thread thread block and 4-thread raking warp.</div>
-     *
-     * \par Performance Considerations
-     * - This variant performs less communication than BLOCK_REDUCE_RAKING_NON_COMMUTATIVE
-     *   and is preferable when the reduction operator is commutative.  This variant
-     *   applies fewer reduction operators  than BLOCK_REDUCE_WARP_REDUCTIONS, and can provide higher overall
-     *   throughput across the GPU when suitably occupied.  However, turn-around latency may be
-     *   higher than to BLOCK_REDUCE_WARP_REDUCTIONS and thus less-desirable
-     *   when the GPU is under-occupied.
-     */
-    BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY,
-
-
-    /**
-     * \par Overview
-     * An efficient "raking" reduction algorithm that supports commutative
-     * (e.g., addition) and non-commutative (e.g., string concatenation) reduction
-     * operators. \blocked.
-     *
-     * \par
-     * Execution is comprised of three phases:
-     * -# Upsweep sequential reduction in registers (if threads contribute more
-     *    than one input each).  Each thread then places the partial reduction
-     *    of its item(s) into shared memory.
-     * -# Upsweep sequential reduction in shared memory.  Threads within a
-     *    single warp rake across segments of shared partial reductions.
-     * -# A warp-synchronous Kogge-Stone style reduction within the raking warp.
-     *
-     * \par
-     * \image html block_reduce.png
-     * <div class="centercaption">\p BLOCK_REDUCE_RAKING data flow for a hypothetical 16-thread thread block and 4-thread raking warp.</div>
-     *
-     * \par Performance Considerations
-     * - This variant performs more communication than BLOCK_REDUCE_RAKING
-     *   and is only preferable when the reduction operator is non-commutative.  This variant
-     *   applies fewer reduction operators than BLOCK_REDUCE_WARP_REDUCTIONS, and can provide higher overall
-     *   throughput across the GPU when suitably occupied.  However, turn-around latency may be
-     *   higher than to BLOCK_REDUCE_WARP_REDUCTIONS and thus less-desirable
-     *   when the GPU is under-occupied.
-     */
-    BLOCK_REDUCE_RAKING,
-
-
-    /**
-     * \par Overview
-     * A quick "tiled warp-reductions" reduction algorithm that supports commutative
-     * (e.g., addition) and non-commutative (e.g., string concatenation) reduction
-     * operators.
-     *
-     * \par
-     * Execution is comprised of four phases:
-     * -# Upsweep sequential reduction in registers (if threads contribute more
-     *    than one input each).  Each thread then places the partial reduction
-     *    of its item(s) into shared memory.
-     * -# Compute a shallow, but inefficient warp-synchronous Kogge-Stone style
-     *    reduction within each warp.
-     * -# A propagation phase where the warp reduction outputs in each warp are
-     *    updated with the aggregate from each preceding warp.
-     *
-     * \par
-     * \image html block_scan_warpscans.png
-     * <div class="centercaption">\p BLOCK_REDUCE_WARP_REDUCTIONS data flow for a hypothetical 16-thread thread block and 4-thread raking warp.</div>
-     *
-     * \par Performance Considerations
-     * - This variant applies more reduction operators than BLOCK_REDUCE_RAKING
-     *   or BLOCK_REDUCE_RAKING_NON_COMMUTATIVE, which may result in lower overall
-     *   throughput across the GPU.  However turn-around latency may be lower and
-     *   thus useful when the GPU is under-occupied.
-     */
-    BLOCK_REDUCE_WARP_REDUCTIONS,
-};
-
-
-/******************************************************************************
- * Block reduce
- ******************************************************************************/
-
-/**
- * \brief The BlockReduce class provides [<em>collective</em>](index.html#sec0) methods for computing a parallel reduction of items partitioned across a CUDA thread block. ![](reduce_logo.png)
- * \ingroup BlockModule
- *
- * \tparam T                Data type being reduced
- * \tparam BLOCK_DIM_X      The thread block length in threads along the X dimension
- * \tparam ALGORITHM        <b>[optional]</b> cub::BlockReduceAlgorithm enumerator specifying the underlying algorithm to use (default: cub::BLOCK_REDUCE_WARP_REDUCTIONS)
- * \tparam BLOCK_DIM_Y      <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
- * \tparam BLOCK_DIM_Z      <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
- * \tparam PTX_ARCH         <b>[optional]</b> \ptxversion
- *
- * \par Overview
- * - A <a href="http://en.wikipedia.org/wiki/Reduce_(higher-order_function)"><em>reduction</em></a> (or <em>fold</em>)
- *   uses a binary combining operator to compute a single aggregate from a list of input elements.
- * - \rowmajor
- * - BlockReduce can be optionally specialized by algorithm to accommodate different latency/throughput workload profiles:
- *   -# <b>cub::BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY</b>.  An efficient "raking" reduction algorithm that only supports commutative reduction operators. [More...](\ref cub::BlockReduceAlgorithm)
- *   -# <b>cub::BLOCK_REDUCE_RAKING</b>.  An efficient "raking" reduction algorithm that supports commutative and non-commutative reduction operators. [More...](\ref cub::BlockReduceAlgorithm)
- *   -# <b>cub::BLOCK_REDUCE_WARP_REDUCTIONS</b>.  A quick "tiled warp-reductions" reduction algorithm that supports commutative and non-commutative reduction operators. [More...](\ref cub::BlockReduceAlgorithm)
- *
- * \par Performance Considerations
- * - \granularity
- * - Very efficient (only one synchronization barrier).
- * - Incurs zero bank conflicts for most types
- * - Computation is slightly more efficient (i.e., having lower instruction overhead) for:
- *   - Summation (<b><em>vs.</em></b> generic reduction)
- *   - \p BLOCK_THREADS is a multiple of the architecture's warp size
- *   - Every thread has a valid input (i.e., full <b><em>vs.</em></b> partial-tiles)
- * - See cub::BlockReduceAlgorithm for performance details regarding algorithmic alternatives
- *
- * \par A Simple Example
- * \blockcollective{BlockReduce}
- * \par
- * The code snippet below illustrates a sum reduction of 512 integer items that
- * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
- * where each thread owns 4 consecutive items.
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/block/block_reduce.cuh>
- *
- * __global__ void ExampleKernel(...)
- * {
- *     // Specialize BlockReduce for a 1D block of 128 threads on type int
- *     typedef cub::BlockReduce<int, 128> BlockReduce;
- *
- *     // Allocate shared memory for BlockReduce
- *     __shared__ typename BlockReduce::TempStorage temp_storage;
- *
- *     // Obtain a segment of consecutive items that are blocked across threads
- *     int thread_data[4];
- *     ...
- *
- *     // Compute the block-wide sum for thread0
- *     int aggregate = BlockReduce(temp_storage).Sum(thread_data);
- *
- * \endcode
- *
- */
-template <
-    typename                T,
-    int                     BLOCK_DIM_X,
-    BlockReduceAlgorithm    ALGORITHM       = BLOCK_REDUCE_WARP_REDUCTIONS,
-    int                     BLOCK_DIM_Y     = 1,
-    int                     BLOCK_DIM_Z     = 1,
-    int                     PTX_ARCH        = CUB_PTX_ARCH>
-class BlockReduce
-{
-private:
-
-    /******************************************************************************
-     * Constants and type definitions
-     ******************************************************************************/
-
-    /// Constants
-    enum
-    {
-        /// The thread block size in threads
-        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-    };
-
-    typedef BlockReduceWarpReductions<T, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH>           WarpReductions;
-    typedef BlockReduceRakingCommutativeOnly<T, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH>    RakingCommutativeOnly;
-    typedef BlockReduceRaking<T, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH>                   Raking;
-
-    /// Internal specialization type
-    typedef typename If<(ALGORITHM == BLOCK_REDUCE_WARP_REDUCTIONS),
-        WarpReductions,
-        typename If<(ALGORITHM == BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY),
-            RakingCommutativeOnly,
-            Raking>::Type>::Type InternalBlockReduce;     // BlockReduceRaking
-
-    /// Shared memory storage layout type for BlockReduce
-    typedef typename InternalBlockReduce::TempStorage _TempStorage;
-
-
-    /******************************************************************************
-     * Utility methods
-     ******************************************************************************/
-
-    /// Internal storage allocator
-    __device__ __forceinline__ _TempStorage& PrivateStorage()
-    {
-        __shared__ _TempStorage private_storage;
-        return private_storage;
-    }
-
-
-    /******************************************************************************
-     * Thread fields
-     ******************************************************************************/
-
-    /// Shared storage reference
-    _TempStorage &temp_storage;
-
-    /// Linear thread-id
-    unsigned int linear_tid;
-
-
-public:
-
-    /// \smemstorage{BlockReduce}
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    /******************************************************************//**
-     * \name Collective constructors
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
-     */
-    __device__ __forceinline__ BlockReduce()
-    :
-        temp_storage(PrivateStorage()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    /**
-     * \brief Collective constructor using the specified memory allocation as temporary storage.
-     */
-    __device__ __forceinline__ BlockReduce(
-        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Generic reductions
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Computes a block-wide reduction for thread<sub>0</sub> using the specified binary reduction functor.  Each thread contributes one input element.
-     *
-     * \par
-     * - The return value is undefined in threads other than thread<sub>0</sub>.
-     * - \rowmajor
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a max reduction of 128 integer items that
-     * are partitioned across 128 threads.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_reduce.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockReduce for a 1D block of 128 threads on type int
-     *     typedef cub::BlockReduce<int, 128> BlockReduce;
-     *
-     *     // Allocate shared memory for BlockReduce
-     *     __shared__ typename BlockReduce::TempStorage temp_storage;
-     *
-     *     // Each thread obtains an input item
-     *     int thread_data;
-     *     ...
-     *
-     *     // Compute the block-wide max for thread0
-     *     int aggregate = BlockReduce(temp_storage).Reduce(thread_data, cub::Max());
-     *
-     * \endcode
-     *
-     * \tparam ReductionOp          <b>[inferred]</b> Binary reduction functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <typename ReductionOp>
-    __device__ __forceinline__ T Reduce(
-        T               input,                      ///< [in] Calling thread's input
-        ReductionOp     reduction_op)               ///< [in] Binary reduction functor 
-    {
-        return InternalBlockReduce(temp_storage).template Reduce<true>(input, BLOCK_THREADS, reduction_op);
-    }
-
-
-    /**
-     * \brief Computes a block-wide reduction for thread<sub>0</sub> using the specified binary reduction functor.  Each thread contributes an array of consecutive input elements.
-     *
-     * \par
-     * - The return value is undefined in threads other than thread<sub>0</sub>.
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a max reduction of 512 integer items that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_reduce.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockReduce for a 1D block of 128 threads on type int
-     *     typedef cub::BlockReduce<int, 128> BlockReduce;
-     *
-     *     // Allocate shared memory for BlockReduce
-     *     __shared__ typename BlockReduce::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Compute the block-wide max for thread0
-     *     int aggregate = BlockReduce(temp_storage).Reduce(thread_data, cub::Max());
-     *
-     * \endcode
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam ReductionOp          <b>[inferred]</b> Binary reduction functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <
-        int ITEMS_PER_THREAD,
-        typename ReductionOp>
-    __device__ __forceinline__ T Reduce(
-        T               (&inputs)[ITEMS_PER_THREAD],    ///< [in] Calling thread's input segment
-        ReductionOp     reduction_op)                   ///< [in] Binary reduction functor 
-    {
-        // Reduce partials
-        T partial = internal::ThreadReduce(inputs, reduction_op);
-        return Reduce(partial, reduction_op);
-    }
-
-
-    /**
-     * \brief Computes a block-wide reduction for thread<sub>0</sub> using the specified binary reduction functor.  The first \p num_valid threads each contribute one input element.
-     *
-     * \par
-     * - The return value is undefined in threads other than thread<sub>0</sub>.
-     * - \rowmajor
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a max reduction of a partially-full tile of integer items that
-     * are partitioned across 128 threads.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_reduce.cuh>
-     *
-     * __global__ void ExampleKernel(int num_valid, ...)
-     * {
-     *     // Specialize BlockReduce for a 1D block of 128 threads on type int
-     *     typedef cub::BlockReduce<int, 128> BlockReduce;
-     *
-     *     // Allocate shared memory for BlockReduce
-     *     __shared__ typename BlockReduce::TempStorage temp_storage;
-     *
-     *     // Each thread obtains an input item
-     *     int thread_data;
-     *     if (threadIdx.x < num_valid) thread_data = ...
-     *
-     *     // Compute the block-wide max for thread0
-     *     int aggregate = BlockReduce(temp_storage).Reduce(thread_data, cub::Max(), num_valid);
-     *
-     * \endcode
-     *
-     * \tparam ReductionOp          <b>[inferred]</b> Binary reduction functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <typename ReductionOp>
-    __device__ __forceinline__ T Reduce(
-        T                   input,                  ///< [in] Calling thread's input
-        ReductionOp         reduction_op,           ///< [in] Binary reduction functor 
-        int                 num_valid)              ///< [in] Number of threads containing valid elements (may be less than BLOCK_THREADS)
-    {
-        // Determine if we scan skip bounds checking
-        if (num_valid >= BLOCK_THREADS)
-        {
-            return InternalBlockReduce(temp_storage).template Reduce<true>(input, num_valid, reduction_op);
-        }
-        else
-        {
-            return InternalBlockReduce(temp_storage).template Reduce<false>(input, num_valid, reduction_op);
-        }
-    }
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Summation reductions
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Computes a block-wide reduction for thread<sub>0</sub> using addition (+) as the reduction operator.  Each thread contributes one input element.
-     *
-     * \par
-     * - The return value is undefined in threads other than thread<sub>0</sub>.
-     * - \rowmajor
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a sum reduction of 128 integer items that
-     * are partitioned across 128 threads.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_reduce.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockReduce for a 1D block of 128 threads on type int
-     *     typedef cub::BlockReduce<int, 128> BlockReduce;
-     *
-     *     // Allocate shared memory for BlockReduce
-     *     __shared__ typename BlockReduce::TempStorage temp_storage;
-     *
-     *     // Each thread obtains an input item
-     *     int thread_data;
-     *     ...
-     *
-     *     // Compute the block-wide sum for thread0
-     *     int aggregate = BlockReduce(temp_storage).Sum(thread_data);
-     *
-     * \endcode
-     *
-     */
-    __device__ __forceinline__ T Sum(
-        T   input)                      ///< [in] Calling thread's input
-    {
-        return InternalBlockReduce(temp_storage).template Sum<true>(input, BLOCK_THREADS);
-    }
-
-    /**
-     * \brief Computes a block-wide reduction for thread<sub>0</sub> using addition (+) as the reduction operator.  Each thread contributes an array of consecutive input elements.
-     *
-     * \par
-     * - The return value is undefined in threads other than thread<sub>0</sub>.
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a sum reduction of 512 integer items that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_reduce.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockReduce for a 1D block of 128 threads on type int
-     *     typedef cub::BlockReduce<int, 128> BlockReduce;
-     *
-     *     // Allocate shared memory for BlockReduce
-     *     __shared__ typename BlockReduce::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Compute the block-wide sum for thread0
-     *     int aggregate = BlockReduce(temp_storage).Sum(thread_data);
-     *
-     * \endcode
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     */
-    template <int ITEMS_PER_THREAD>
-    __device__ __forceinline__ T Sum(
-        T   (&inputs)[ITEMS_PER_THREAD])    ///< [in] Calling thread's input segment
-    {
-        // Reduce partials
-        T partial = internal::ThreadReduce(inputs, cub::Sum());
-        return Sum(partial);
-    }
-
-
-    /**
-     * \brief Computes a block-wide reduction for thread<sub>0</sub> using addition (+) as the reduction operator.  The first \p num_valid threads each contribute one input element.
-     *
-     * \par
-     * - The return value is undefined in threads other than thread<sub>0</sub>.
-     * - \rowmajor
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a sum reduction of a partially-full tile of integer items that
-     * are partitioned across 128 threads.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_reduce.cuh>
-     *
-     * __global__ void ExampleKernel(int num_valid, ...)
-     * {
-     *     // Specialize BlockReduce for a 1D block of 128 threads on type int
-     *     typedef cub::BlockReduce<int, 128> BlockReduce;
-     *
-     *     // Allocate shared memory for BlockReduce
-     *     __shared__ typename BlockReduce::TempStorage temp_storage;
-     *
-     *     // Each thread obtains an input item (up to num_items)
-     *     int thread_data;
-     *     if (threadIdx.x < num_valid)
-     *         thread_data = ...
-     *
-     *     // Compute the block-wide sum for thread0
-     *     int aggregate = BlockReduce(temp_storage).Sum(thread_data, num_valid);
-     *
-     * \endcode
-     *
-     */
-    __device__ __forceinline__ T Sum(
-        T   input,                  ///< [in] Calling thread's input
-        int num_valid)              ///< [in] Number of threads containing valid elements (may be less than BLOCK_THREADS)
-    {
-        // Determine if we scan skip bounds checking
-        if (num_valid >= BLOCK_THREADS)
-        {
-            return InternalBlockReduce(temp_storage).template Sum<true>(input, num_valid);
-        }
-        else
-        {
-            return InternalBlockReduce(temp_storage).template Sum<false>(input, num_valid);
-        }
-    }
-
-
-    //@}  end member group
-};
-
-/**
- * \example example_block_reduce.cu
- */
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/block/block_scan.cuh b/pfsimulator/third_party/cub/block/block_scan.cuh
deleted file mode 100644
index 27ea7ed40..000000000
--- a/pfsimulator/third_party/cub/block/block_scan.cuh
+++ /dev/null
@@ -1,2126 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * The cub::BlockScan class provides [<em>collective</em>](index.html#sec0) methods for computing a parallel prefix sum/scan of items partitioned across a CUDA thread block.
- */
-
-#pragma once
-
-#include "specializations/block_scan_raking.cuh"
-#include "specializations/block_scan_warp_scans.cuh"
-#include "../util_arch.cuh"
-#include "../util_type.cuh"
-#include "../util_ptx.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/******************************************************************************
- * Algorithmic variants
- ******************************************************************************/
-
-/**
- * \brief BlockScanAlgorithm enumerates alternative algorithms for cub::BlockScan to compute a parallel prefix scan across a CUDA thread block.
- */
-enum BlockScanAlgorithm
-{
-
-    /**
-     * \par Overview
-     * An efficient "raking reduce-then-scan" prefix scan algorithm.  Execution is comprised of five phases:
-     * -# Upsweep sequential reduction in registers (if threads contribute more than one input each).  Each thread then places the partial reduction of its item(s) into shared memory.
-     * -# Upsweep sequential reduction in shared memory.  Threads within a single warp rake across segments of shared partial reductions.
-     * -# A warp-synchronous Kogge-Stone style exclusive scan within the raking warp.
-     * -# Downsweep sequential exclusive scan in shared memory.  Threads within a single warp rake across segments of shared partial reductions, seeded with the warp-scan output.
-     * -# Downsweep sequential scan in registers (if threads contribute more than one input), seeded with the raking scan output.
-     *
-     * \par
-     * \image html block_scan_raking.png
-     * <div class="centercaption">\p BLOCK_SCAN_RAKING data flow for a hypothetical 16-thread thread block and 4-thread raking warp.</div>
-     *
-     * \par Performance Considerations
-     * - Although this variant may suffer longer turnaround latencies when the
-     *   GPU is under-occupied, it can often provide higher overall throughput
-     *   across the GPU when suitably occupied.
-     */
-    BLOCK_SCAN_RAKING,
-
-
-    /**
-     * \par Overview
-     * Similar to cub::BLOCK_SCAN_RAKING, but with fewer shared memory reads at
-     * the expense of higher register pressure.  Raking threads preserve their
-     * "upsweep" segment of values in registers while performing warp-synchronous
-     * scan, allowing the "downsweep" not to re-read them from shared memory.
-     */
-    BLOCK_SCAN_RAKING_MEMOIZE,
-
-
-    /**
-     * \par Overview
-     * A quick "tiled warpscans" prefix scan algorithm.  Execution is comprised of four phases:
-     * -# Upsweep sequential reduction in registers (if threads contribute more than one input each).  Each thread then places the partial reduction of its item(s) into shared memory.
-     * -# Compute a shallow, but inefficient warp-synchronous Kogge-Stone style scan within each warp.
-     * -# A propagation phase where the warp scan outputs in each warp are updated with the aggregate from each preceding warp.
-     * -# Downsweep sequential scan in registers (if threads contribute more than one input), seeded with the raking scan output.
-     *
-     * \par
-     * \image html block_scan_warpscans.png
-     * <div class="centercaption">\p BLOCK_SCAN_WARP_SCANS data flow for a hypothetical 16-thread thread block and 4-thread raking warp.</div>
-     *
-     * \par Performance Considerations
-     * - Although this variant may suffer lower overall throughput across the
-     *   GPU because due to a heavy reliance on inefficient warpscans, it can
-     *   often provide lower turnaround latencies when the GPU is under-occupied.
-     */
-    BLOCK_SCAN_WARP_SCANS,
-};
-
-
-/******************************************************************************
- * Block scan
- ******************************************************************************/
-
-/**
- * \brief The BlockScan class provides [<em>collective</em>](index.html#sec0) methods for computing a parallel prefix sum/scan of items partitioned across a CUDA thread block. ![](block_scan_logo.png)
- * \ingroup BlockModule
- *
- * \tparam T                Data type being scanned
- * \tparam BLOCK_DIM_X      The thread block length in threads along the X dimension
- * \tparam ALGORITHM        <b>[optional]</b> cub::BlockScanAlgorithm enumerator specifying the underlying algorithm to use (default: cub::BLOCK_SCAN_RAKING)
- * \tparam BLOCK_DIM_Y      <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
- * \tparam BLOCK_DIM_Z      <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
- * \tparam PTX_ARCH         <b>[optional]</b> \ptxversion
- *
- * \par Overview
- * - Given a list of input elements and a binary reduction operator, a [<em>prefix scan</em>](http://en.wikipedia.org/wiki/Prefix_sum)
- *   produces an output list where each element is computed to be the reduction
- *   of the elements occurring earlier in the input list.  <em>Prefix sum</em>
- *   connotes a prefix scan with the addition operator. The term \em inclusive indicates
- *   that the <em>i</em><sup>th</sup> output reduction incorporates the <em>i</em><sup>th</sup> input.
- *   The term \em exclusive indicates the <em>i</em><sup>th</sup> input is not incorporated into
- *   the <em>i</em><sup>th</sup> output reduction.
- * - \rowmajor
- * - BlockScan can be optionally specialized by algorithm to accommodate different workload profiles:
- *   -# <b>cub::BLOCK_SCAN_RAKING</b>.  An efficient (high throughput) "raking reduce-then-scan" prefix scan algorithm. [More...](\ref cub::BlockScanAlgorithm)
- *   -# <b>cub::BLOCK_SCAN_RAKING_MEMOIZE</b>.  Similar to cub::BLOCK_SCAN_RAKING, but having higher throughput at the expense of additional register pressure for intermediate storage. [More...](\ref cub::BlockScanAlgorithm)
- *   -# <b>cub::BLOCK_SCAN_WARP_SCANS</b>.  A quick (low latency) "tiled warpscans" prefix scan algorithm. [More...](\ref cub::BlockScanAlgorithm)
- *
- * \par Performance Considerations
- * - \granularity
- * - Uses special instructions when applicable (e.g., warp \p SHFL)
- * - Uses synchronization-free communication between warp lanes when applicable
- * - Invokes a minimal number of minimal block-wide synchronization barriers (only
- *   one or two depending on algorithm selection)
- * - Incurs zero bank conflicts for most types
- * - Computation is slightly more efficient (i.e., having lower instruction overhead) for:
- *   - Prefix sum variants (<b><em>vs.</em></b> generic scan)
- *   - \blocksize
- * - See cub::BlockScanAlgorithm for performance details regarding algorithmic alternatives
- *
- * \par A Simple Example
- * \blockcollective{BlockScan}
- * \par
- * The code snippet below illustrates an exclusive prefix sum of 512 integer items that
- * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
- * where each thread owns 4 consecutive items.
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
- *
- * __global__ void ExampleKernel(...)
- * {
- *     // Specialize BlockScan for a 1D block of 128 threads on type int
- *     typedef cub::BlockScan<int, 128> BlockScan;
- *
- *     // Allocate shared memory for BlockScan
- *     __shared__ typename BlockScan::TempStorage temp_storage;
- *
- *     // Obtain a segment of consecutive items that are blocked across threads
- *     int thread_data[4];
- *     ...
- *
- *     // Collectively compute the block-wide exclusive prefix sum
- *     BlockScan(temp_storage).ExclusiveSum(thread_data, thread_data);
- *
- * \endcode
- * \par
- * Suppose the set of input \p thread_data across the block of threads is
- * <tt>{[1,1,1,1], [1,1,1,1], ..., [1,1,1,1]}</tt>.
- * The corresponding output \p thread_data in those threads will be
- * <tt>{[0,1,2,3], [4,5,6,7], ..., [508,509,510,511]}</tt>.
- *
- */
-template <
-    typename            T,
-    int                 BLOCK_DIM_X,
-    BlockScanAlgorithm  ALGORITHM       = BLOCK_SCAN_RAKING,
-    int                 BLOCK_DIM_Y     = 1,
-    int                 BLOCK_DIM_Z     = 1,
-    int                 PTX_ARCH        = CUB_PTX_ARCH>
-class BlockScan
-{
-private:
-
-    /******************************************************************************
-     * Constants and type definitions
-     ******************************************************************************/
-
-    /// Constants
-    enum
-    {
-        /// The thread block size in threads
-        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-    };
-
-    /**
-     * Ensure the template parameterization meets the requirements of the
-     * specified algorithm. Currently, the BLOCK_SCAN_WARP_SCANS policy
-     * cannot be used with thread block sizes not a multiple of the
-     * architectural warp size.
-     */
-    static const BlockScanAlgorithm SAFE_ALGORITHM =
-        ((ALGORITHM == BLOCK_SCAN_WARP_SCANS) && (BLOCK_THREADS % CUB_WARP_THREADS(PTX_ARCH) != 0)) ?
-            BLOCK_SCAN_RAKING :
-            ALGORITHM;
-
-    typedef BlockScanWarpScans<T, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> WarpScans;
-    typedef BlockScanRaking<T, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z, (SAFE_ALGORITHM == BLOCK_SCAN_RAKING_MEMOIZE), PTX_ARCH> Raking;
-
-    /// Define the delegate type for the desired algorithm
-    typedef typename If<(SAFE_ALGORITHM == BLOCK_SCAN_WARP_SCANS),
-        WarpScans,
-        Raking>::Type InternalBlockScan;
-
-    /// Shared memory storage layout type for BlockScan
-    typedef typename InternalBlockScan::TempStorage _TempStorage;
-
-
-    /******************************************************************************
-     * Thread fields
-     ******************************************************************************/
-
-    /// Shared storage reference
-    _TempStorage &temp_storage;
-
-    /// Linear thread-id
-    unsigned int linear_tid;
-
-
-    /******************************************************************************
-     * Utility methods
-     ******************************************************************************/
-
-    /// Internal storage allocator
-    __device__ __forceinline__ _TempStorage& PrivateStorage()
-    {
-        __shared__ _TempStorage private_storage;
-        return private_storage;
-    }
-
-
-    /******************************************************************************
-     * Public types
-     ******************************************************************************/
-public:
-
-    /// \smemstorage{BlockScan}
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    /******************************************************************//**
-     * \name Collective constructors
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
-     */
-    __device__ __forceinline__ BlockScan()
-    :
-        temp_storage(PrivateStorage()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    /**
-     * \brief Collective constructor using the specified memory allocation as temporary storage.
-     */
-    __device__ __forceinline__ BlockScan(
-        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Exclusive prefix sum operations
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Computes an exclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes one input element.  The value of 0 is applied as the initial value, and is assigned to \p output in <em>thread</em><sub>0</sub>.
-     *
-     * \par
-     * - \identityzero
-     * - \rowmajor
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates an exclusive prefix sum of 128 integer items that
-     * are partitioned across 128 threads.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads on type int
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Obtain input item for each thread
-     *     int thread_data;
-     *     ...
-     *
-     *     // Collectively compute the block-wide exclusive prefix sum
-     *     BlockScan(temp_storage).ExclusiveSum(thread_data, thread_data);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>1, 1, ..., 1</tt>.  The
-     * corresponding output \p thread_data in those threads will be <tt>0, 1, ..., 127</tt>.
-     *
-     */
-    __device__ __forceinline__ void ExclusiveSum(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &output)                        ///< [out] Calling thread's output item (may be aliased to \p input)
-    {
-        T initial_value = 0;
-        ExclusiveScan(input, output, initial_value, cub::Sum());
-    }
-
-
-    /**
-     * \brief Computes an exclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes one input element.  The value of 0 is applied as the initial value, and is assigned to \p output in <em>thread</em><sub>0</sub>.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-     *
-     * \par
-     * - \identityzero
-     * - \rowmajor
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates an exclusive prefix sum of 128 integer items that
-     * are partitioned across 128 threads.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads on type int
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Obtain input item for each thread
-     *     int thread_data;
-     *     ...
-     *
-     *     // Collectively compute the block-wide exclusive prefix sum
-     *     int block_aggregate;
-     *     BlockScan(temp_storage).ExclusiveSum(thread_data, thread_data, block_aggregate);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>1, 1, ..., 1</tt>.  The
-     * corresponding output \p thread_data in those threads will be <tt>0, 1, ..., 127</tt>.
-     * Furthermore the value \p 128 will be stored in \p block_aggregate for all threads.
-     *
-     */
-    __device__ __forceinline__ void ExclusiveSum(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
-        T               &block_aggregate)               ///< [out] block-wide aggregate reduction of input items
-    {
-        T initial_value = 0;
-        ExclusiveScan(input, output, initial_value, cub::Sum(), block_aggregate);
-    }
-
-
-    /**
-     * \brief Computes an exclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes one input element.  Instead of using 0 as the block-wide prefix, the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-     *
-     * \par
-     * - \identityzero
-     * - The \p block_prefix_callback_op functor must implement a member function <tt>T operator()(T block_aggregate)</tt>.
-     *   The functor's input parameter \p block_aggregate is the same value also returned by the scan operation.
-     *   The functor will be invoked by the first warp of threads in the block, however only the return value from
-     *   <em>lane</em><sub>0</sub> is applied as the block-wide prefix.  Can be stateful.
-     * - \rowmajor
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a single thread block that progressively
-     * computes an exclusive prefix sum over multiple "tiles" of input using a
-     * prefix functor to maintain a running total between block-wide scans.  Each tile consists
-     * of 128 integer items that are partitioned across 128 threads.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * // A stateful callback functor that maintains a running prefix to be applied
-     * // during consecutive scan operations.
-     * struct BlockPrefixCallbackOp
-     * {
-     *     // Running prefix
-     *     int running_total;
-     *
-     *     // Constructor
-     *     __device__ BlockPrefixCallbackOp(int running_total) : running_total(running_total) {}
-     *
-     *     // Callback operator to be entered by the first warp of threads in the block.
-     *     // Thread-0 is responsible for returning a value for seeding the block-wide scan.
-     *     __device__ int operator()(int block_aggregate)
-     *     {
-     *         int old_prefix = running_total;
-     *         running_total += block_aggregate;
-     *         return old_prefix;
-     *     }
-     * };
-     *
-     * __global__ void ExampleKernel(int *d_data, int num_items, ...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Initialize running total
-     *     BlockPrefixCallbackOp prefix_op(0);
-     *
-     *     // Have the block iterate over segments of items
-     *     for (int block_offset = 0; block_offset < num_items; block_offset += 128)
-     *     {
-     *         // Load a segment of consecutive items that are blocked across threads
-     *         int thread_data = d_data[block_offset];
-     *
-     *         // Collectively compute the block-wide exclusive prefix sum
-     *         BlockScan(temp_storage).ExclusiveSum(
-     *             thread_data, thread_data, prefix_op);
-     *         CTA_SYNC();
-     *
-     *         // Store scanned items to output segment
-     *         d_data[block_offset] = thread_data;
-     *     }
-     * \endcode
-     * \par
-     * Suppose the input \p d_data is <tt>1, 1, 1, 1, 1, 1, 1, 1, ...</tt>.
-     * The corresponding output for the first segment will be <tt>0, 1, ..., 127</tt>.
-     * The output for the second segment will be <tt>128, 129, ..., 255</tt>.
-     *
-     * \tparam BlockPrefixCallbackOp        <b>[inferred]</b> Call-back functor type having member <tt>T operator()(T block_aggregate)</tt>
-     */
-    template <typename BlockPrefixCallbackOp>
-    __device__ __forceinline__ void ExclusiveSum(
-        T                       input,                          ///< [in] Calling thread's input item
-        T                       &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
-        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a block-wide prefix to be applied to the logical input sequence.
-    {
-        ExclusiveScan(input, output, cub::Sum(), block_prefix_callback_op);
-    }
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Exclusive prefix sum operations (multiple data per thread)
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Computes an exclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes an array of consecutive input elements.  The value of 0 is applied as the initial value, and is assigned to \p output[0] in <em>thread</em><sub>0</sub>.
-     *
-     * \par
-     * - \identityzero
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates an exclusive prefix sum of 512 integer items that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads on type int
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Collectively compute the block-wide exclusive prefix sum
-     *     BlockScan(temp_storage).ExclusiveSum(thread_data, thread_data);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{ [1,1,1,1], [1,1,1,1], ..., [1,1,1,1] }</tt>.  The
-     * corresponding output \p thread_data in those threads will be <tt>{ [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }</tt>.
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     */
-    template <int ITEMS_PER_THREAD>
-    __device__ __forceinline__ void ExclusiveSum(
-        T                 (&input)[ITEMS_PER_THREAD],   ///< [in] Calling thread's input items
-        T                 (&output)[ITEMS_PER_THREAD])  ///< [out] Calling thread's output items (may be aliased to \p input)
-    {
-        T initial_value = 0;
-        ExclusiveScan(input, output, initial_value, cub::Sum());
-    }
-
-
-    /**
-     * \brief Computes an exclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes an array of consecutive input elements.  The value of 0 is applied as the initial value, and is assigned to \p output[0] in <em>thread</em><sub>0</sub>.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-     *
-     * \par
-     * - \identityzero
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates an exclusive prefix sum of 512 integer items that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads on type int
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Collectively compute the block-wide exclusive prefix sum
-     *     int block_aggregate;
-     *     BlockScan(temp_storage).ExclusiveSum(thread_data, thread_data, block_aggregate);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{ [1,1,1,1], [1,1,1,1], ..., [1,1,1,1] }</tt>.  The
-     * corresponding output \p thread_data in those threads will be <tt>{ [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }</tt>.
-     * Furthermore the value \p 512 will be stored in \p block_aggregate for all threads.
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     */
-    template <int ITEMS_PER_THREAD>
-    __device__ __forceinline__ void ExclusiveSum(
-        T                 (&input)[ITEMS_PER_THREAD],       ///< [in] Calling thread's input items
-        T                 (&output)[ITEMS_PER_THREAD],      ///< [out] Calling thread's output items (may be aliased to \p input)
-        T                 &block_aggregate)                 ///< [out] block-wide aggregate reduction of input items
-    {
-        // Reduce consecutive thread items in registers
-        T initial_value = 0;
-        ExclusiveScan(input, output, initial_value, cub::Sum(), block_aggregate);
-    }
-
-
-    /**
-     * \brief Computes an exclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes an array of consecutive input elements.  Instead of using 0 as the block-wide prefix, the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-     *
-     * \par
-     * - \identityzero
-     * - The \p block_prefix_callback_op functor must implement a member function <tt>T operator()(T block_aggregate)</tt>.
-     *   The functor's input parameter \p block_aggregate is the same value also returned by the scan operation.
-     *   The functor will be invoked by the first warp of threads in the block, however only the return value from
-     *   <em>lane</em><sub>0</sub> is applied as the block-wide prefix.  Can be stateful.
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a single thread block that progressively
-     * computes an exclusive prefix sum over multiple "tiles" of input using a
-     * prefix functor to maintain a running total between block-wide scans.  Each tile consists
-     * of 512 integer items that are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3)
-     * across 128 threads where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * // A stateful callback functor that maintains a running prefix to be applied
-     * // during consecutive scan operations.
-     * struct BlockPrefixCallbackOp
-     * {
-     *     // Running prefix
-     *     int running_total;
-     *
-     *     // Constructor
-     *     __device__ BlockPrefixCallbackOp(int running_total) : running_total(running_total) {}
-     *
-     *     // Callback operator to be entered by the first warp of threads in the block.
-     *     // Thread-0 is responsible for returning a value for seeding the block-wide scan.
-     *     __device__ int operator()(int block_aggregate)
-     *     {
-     *         int old_prefix = running_total;
-     *         running_total += block_aggregate;
-     *         return old_prefix;
-     *     }
-     * };
-     *
-     * __global__ void ExampleKernel(int *d_data, int num_items, ...)
-     * {
-     *     // Specialize BlockLoad, BlockStore, and BlockScan for a 1D block of 128 threads, 4 ints per thread
-     *     typedef cub::BlockLoad<int*, 128, 4, BLOCK_LOAD_TRANSPOSE>   BlockLoad;
-     *     typedef cub::BlockStore<int, 128, 4, BLOCK_STORE_TRANSPOSE>  BlockStore;
-     *     typedef cub::BlockScan<int, 128>                             BlockScan;
-     *
-     *     // Allocate aliased shared memory for BlockLoad, BlockStore, and BlockScan
-     *     __shared__ union {
-     *         typename BlockLoad::TempStorage     load;
-     *         typename BlockScan::TempStorage     scan;
-     *         typename BlockStore::TempStorage    store;
-     *     } temp_storage;
-     *
-     *     // Initialize running total
-     *     BlockPrefixCallbackOp prefix_op(0);
-     *
-     *     // Have the block iterate over segments of items
-     *     for (int block_offset = 0; block_offset < num_items; block_offset += 128 * 4)
-     *     {
-     *         // Load a segment of consecutive items that are blocked across threads
-     *         int thread_data[4];
-     *         BlockLoad(temp_storage.load).Load(d_data + block_offset, thread_data);
-     *         CTA_SYNC();
-     *
-     *         // Collectively compute the block-wide exclusive prefix sum
-     *         int block_aggregate;
-     *         BlockScan(temp_storage.scan).ExclusiveSum(
-     *             thread_data, thread_data, prefix_op);
-     *         CTA_SYNC();
-     *
-     *         // Store scanned items to output segment
-     *         BlockStore(temp_storage.store).Store(d_data + block_offset, thread_data);
-     *         CTA_SYNC();
-     *     }
-     * \endcode
-     * \par
-     * Suppose the input \p d_data is <tt>1, 1, 1, 1, 1, 1, 1, 1, ...</tt>.
-     * The corresponding output for the first segment will be <tt>0, 1, 2, 3, ..., 510, 511</tt>.
-     * The output for the second segment will be <tt>512, 513, 514, 515, ..., 1022, 1023</tt>.
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam BlockPrefixCallbackOp        <b>[inferred]</b> Call-back functor type having member <tt>T operator()(T block_aggregate)</tt>
-     */
-    template <
-        int ITEMS_PER_THREAD,
-        typename BlockPrefixCallbackOp>
-    __device__ __forceinline__ void ExclusiveSum(
-        T                       (&input)[ITEMS_PER_THREAD],   ///< [in] Calling thread's input items
-        T                       (&output)[ITEMS_PER_THREAD],  ///< [out] Calling thread's output items (may be aliased to \p input)
-        BlockPrefixCallbackOp   &block_prefix_callback_op)    ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a block-wide prefix to be applied to the logical input sequence.
-    {
-        ExclusiveScan(input, output, cub::Sum(), block_prefix_callback_op);
-    }
-
-
-
-    //@}  end member group        // Exclusive prefix sums
-    /******************************************************************//**
-     * \name Exclusive prefix scan operations
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
-     *
-     * \par
-     * - Supports non-commutative scan operators.
-     * - \rowmajor
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates an exclusive prefix max scan of 128 integer items that
-     * are partitioned across 128 threads.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads on type int
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Obtain input item for each thread
-     *     int thread_data;
-     *     ...
-     *
-     *     // Collectively compute the block-wide exclusive prefix max scan
-     *     BlockScan(temp_storage).ExclusiveScan(thread_data, thread_data, INT_MIN, cub::Max());
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>0, -1, 2, -3, ..., 126, -127</tt>.  The
-     * corresponding output \p thread_data in those threads will be <tt>INT_MIN, 0, 0, 2, ..., 124, 126</tt>.
-     *
-     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
-        T               initial_value,                  ///< [in] Initial value to seed the exclusive scan (and is assigned to \p output[0] in <em>thread</em><sub>0</sub>)
-        ScanOp          scan_op)                        ///< [in] Binary scan functor 
-    {
-        InternalBlockScan(temp_storage).ExclusiveScan(input, output, initial_value, scan_op);
-    }
-
-
-    /**
-     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-     *
-     * \par
-     * - Supports non-commutative scan operators.
-     * - \rowmajor
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates an exclusive prefix max scan of 128 integer items that
-     * are partitioned across 128 threads.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads on type int
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Obtain input item for each thread
-     *     int thread_data;
-     *     ...
-     *
-     *     // Collectively compute the block-wide exclusive prefix max scan
-     *     int block_aggregate;
-     *     BlockScan(temp_storage).ExclusiveScan(thread_data, thread_data, INT_MIN, cub::Max(), block_aggregate);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>0, -1, 2, -3, ..., 126, -127</tt>.  The
-     * corresponding output \p thread_data in those threads will be <tt>INT_MIN, 0, 0, 2, ..., 124, 126</tt>.
-     * Furthermore the value \p 126 will be stored in \p block_aggregate for all threads.
-     *
-     * \tparam ScanOp   <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,              ///< [in] Calling thread's input items
-        T               &output,            ///< [out] Calling thread's output items (may be aliased to \p input)
-        T               initial_value,      ///< [in] Initial value to seed the exclusive scan (and is assigned to \p output[0] in <em>thread</em><sub>0</sub>)
-        ScanOp          scan_op,            ///< [in] Binary scan functor 
-        T               &block_aggregate)   ///< [out] block-wide aggregate reduction of input items
-    {
-        InternalBlockScan(temp_storage).ExclusiveScan(input, output, initial_value, scan_op, block_aggregate);
-    }
-
-
-    /**
-     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-     *
-     * \par
-     * - The \p block_prefix_callback_op functor must implement a member function <tt>T operator()(T block_aggregate)</tt>.
-     *   The functor's input parameter \p block_aggregate is the same value also returned by the scan operation.
-     *   The functor will be invoked by the first warp of threads in the block, however only the return value from
-     *   <em>lane</em><sub>0</sub> is applied as the block-wide prefix.  Can be stateful.
-     * - Supports non-commutative scan operators.
-     * - \rowmajor
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a single thread block that progressively
-     * computes an exclusive prefix max scan over multiple "tiles" of input using a
-     * prefix functor to maintain a running total between block-wide scans.  Each tile consists
-     * of 128 integer items that are partitioned across 128 threads.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * // A stateful callback functor that maintains a running prefix to be applied
-     * // during consecutive scan operations.
-     * struct BlockPrefixCallbackOp
-     * {
-     *     // Running prefix
-     *     int running_total;
-     *
-     *     // Constructor
-     *     __device__ BlockPrefixCallbackOp(int running_total) : running_total(running_total) {}
-     *
-     *     // Callback operator to be entered by the first warp of threads in the block.
-     *     // Thread-0 is responsible for returning a value for seeding the block-wide scan.
-     *     __device__ int operator()(int block_aggregate)
-     *     {
-     *         int old_prefix = running_total;
-     *         running_total = (block_aggregate > old_prefix) ? block_aggregate : old_prefix;
-     *         return old_prefix;
-     *     }
-     * };
-     *
-     * __global__ void ExampleKernel(int *d_data, int num_items, ...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Initialize running total
-     *     BlockPrefixCallbackOp prefix_op(INT_MIN);
-     *
-     *     // Have the block iterate over segments of items
-     *     for (int block_offset = 0; block_offset < num_items; block_offset += 128)
-     *     {
-     *         // Load a segment of consecutive items that are blocked across threads
-     *         int thread_data = d_data[block_offset];
-     *
-     *         // Collectively compute the block-wide exclusive prefix max scan
-     *         BlockScan(temp_storage).ExclusiveScan(
-     *             thread_data, thread_data, INT_MIN, cub::Max(), prefix_op);
-     *         CTA_SYNC();
-     *
-     *         // Store scanned items to output segment
-     *         d_data[block_offset] = thread_data;
-     *     }
-     * \endcode
-     * \par
-     * Suppose the input \p d_data is <tt>0, -1, 2, -3, 4, -5, ...</tt>.
-     * The corresponding output for the first segment will be <tt>INT_MIN, 0, 0, 2, ..., 124, 126</tt>.
-     * The output for the second segment will be <tt>126, 128, 128, 130, ..., 252, 254</tt>.
-     *
-     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     * \tparam BlockPrefixCallbackOp        <b>[inferred]</b> Call-back functor type having member <tt>T operator()(T block_aggregate)</tt>
-     */
-    template <
-        typename ScanOp,
-        typename BlockPrefixCallbackOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T                       input,                          ///< [in] Calling thread's input item
-        T                       &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp                  scan_op,                        ///< [in] Binary scan functor 
-        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a block-wide prefix to be applied to the logical input sequence.
-    {
-        InternalBlockScan(temp_storage).ExclusiveScan(input, output, scan_op, block_prefix_callback_op);
-    }
-
-
-    //@}  end member group        // Inclusive prefix sums
-    /******************************************************************//**
-     * \name Exclusive prefix scan operations (multiple data per thread)
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes an array of consecutive input elements.
-     *
-     * \par
-     * - Supports non-commutative scan operators.
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates an exclusive prefix max scan of 512 integer items that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads on type int
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Collectively compute the block-wide exclusive prefix max scan
-     *     BlockScan(temp_storage).ExclusiveScan(thread_data, thread_data, INT_MIN, cub::Max());
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is
-     * <tt>{ [0,-1,2,-3], [4,-5,6,-7], ..., [508,-509,510,-511] }</tt>.
-     * The corresponding output \p thread_data in those threads will be
-     * <tt>{ [INT_MIN,0,0,2], [2,4,4,6], ..., [506,508,508,510] }</tt>.
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T                 (&input)[ITEMS_PER_THREAD],   ///< [in] Calling thread's input items
-        T                 (&output)[ITEMS_PER_THREAD],  ///< [out] Calling thread's output items (may be aliased to \p input)
-        T                 initial_value,                ///< [in] Initial value to seed the exclusive scan (and is assigned to \p output[0] in <em>thread</em><sub>0</sub>)
-        ScanOp            scan_op)                      ///< [in] Binary scan functor
-    {
-        // Reduce consecutive thread items in registers
-        T thread_prefix = internal::ThreadReduce(input, scan_op);
-
-        // Exclusive thread block-scan
-        ExclusiveScan(thread_prefix, thread_prefix, initial_value, scan_op);
-
-        // Exclusive scan in registers with prefix as seed
-        internal::ThreadScanExclusive(input, output, scan_op, thread_prefix);
-    }
-
-
-    /**
-     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes an array of consecutive input elements.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-     *
-     * \par
-     * - Supports non-commutative scan operators.
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates an exclusive prefix max scan of 512 integer items that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads on type int
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Collectively compute the block-wide exclusive prefix max scan
-     *     int block_aggregate;
-     *     BlockScan(temp_storage).ExclusiveScan(thread_data, thread_data, INT_MIN, cub::Max(), block_aggregate);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{ [0,-1,2,-3], [4,-5,6,-7], ..., [508,-509,510,-511] }</tt>.  The
-     * corresponding output \p thread_data in those threads will be <tt>{ [INT_MIN,0,0,2], [2,4,4,6], ..., [506,508,508,510] }</tt>.
-     * Furthermore the value \p 510 will be stored in \p block_aggregate for all threads.
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T                 (&input)[ITEMS_PER_THREAD],   ///< [in] Calling thread's input items
-        T                 (&output)[ITEMS_PER_THREAD],  ///< [out] Calling thread's output items (may be aliased to \p input)
-        T                 initial_value,                ///< [in] Initial value to seed the exclusive scan (and is assigned to \p output[0] in <em>thread</em><sub>0</sub>)
-        ScanOp            scan_op,                      ///< [in] Binary scan functor
-        T                 &block_aggregate)             ///< [out] block-wide aggregate reduction of input items
-    {
-        // Reduce consecutive thread items in registers
-        T thread_prefix = internal::ThreadReduce(input, scan_op);
-
-        // Exclusive thread block-scan
-        ExclusiveScan(thread_prefix, thread_prefix, initial_value, scan_op, block_aggregate);
-
-        // Exclusive scan in registers with prefix as seed
-        internal::ThreadScanExclusive(input, output, scan_op, thread_prefix);
-    }
-
-
-    /**
-     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes an array of consecutive input elements.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-     *
-     * \par
-     * - The \p block_prefix_callback_op functor must implement a member function <tt>T operator()(T block_aggregate)</tt>.
-     *   The functor's input parameter \p block_aggregate is the same value also returned by the scan operation.
-     *   The functor will be invoked by the first warp of threads in the block, however only the return value from
-     *   <em>lane</em><sub>0</sub> is applied as the block-wide prefix.  Can be stateful.
-     * - Supports non-commutative scan operators.
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a single thread block that progressively
-     * computes an exclusive prefix max scan over multiple "tiles" of input using a
-     * prefix functor to maintain a running total between block-wide scans.  Each tile consists
-     * of 128 integer items that are partitioned across 128 threads.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * // A stateful callback functor that maintains a running prefix to be applied
-     * // during consecutive scan operations.
-     * struct BlockPrefixCallbackOp
-     * {
-     *     // Running prefix
-     *     int running_total;
-     *
-     *     // Constructor
-     *     __device__ BlockPrefixCallbackOp(int running_total) : running_total(running_total) {}
-     *
-     *     // Callback operator to be entered by the first warp of threads in the block.
-     *     // Thread-0 is responsible for returning a value for seeding the block-wide scan.
-     *     __device__ int operator()(int block_aggregate)
-     *     {
-     *         int old_prefix = running_total;
-     *         running_total = (block_aggregate > old_prefix) ? block_aggregate : old_prefix;
-     *         return old_prefix;
-     *     }
-     * };
-     *
-     * __global__ void ExampleKernel(int *d_data, int num_items, ...)
-     * {
-     *     // Specialize BlockLoad, BlockStore, and BlockScan for a 1D block of 128 threads, 4 ints per thread
-     *     typedef cub::BlockLoad<int*, 128, 4, BLOCK_LOAD_TRANSPOSE>   BlockLoad;
-     *     typedef cub::BlockStore<int, 128, 4, BLOCK_STORE_TRANSPOSE>  BlockStore;
-     *     typedef cub::BlockScan<int, 128>                             BlockScan;
-     *
-     *     // Allocate aliased shared memory for BlockLoad, BlockStore, and BlockScan
-     *     __shared__ union {
-     *         typename BlockLoad::TempStorage     load;
-     *         typename BlockScan::TempStorage     scan;
-     *         typename BlockStore::TempStorage    store;
-     *     } temp_storage;
-     *
-     *     // Initialize running total
-     *     BlockPrefixCallbackOp prefix_op(0);
-     *
-     *     // Have the block iterate over segments of items
-     *     for (int block_offset = 0; block_offset < num_items; block_offset += 128 * 4)
-     *     {
-     *         // Load a segment of consecutive items that are blocked across threads
-     *         int thread_data[4];
-     *         BlockLoad(temp_storage.load).Load(d_data + block_offset, thread_data);
-     *         CTA_SYNC();
-     *
-     *         // Collectively compute the block-wide exclusive prefix max scan
-     *         BlockScan(temp_storage.scan).ExclusiveScan(
-     *             thread_data, thread_data, INT_MIN, cub::Max(), prefix_op);
-     *         CTA_SYNC();
-     *
-     *         // Store scanned items to output segment
-     *         BlockStore(temp_storage.store).Store(d_data + block_offset, thread_data);
-     *         CTA_SYNC();
-     *     }
-     * \endcode
-     * \par
-     * Suppose the input \p d_data is <tt>0, -1, 2, -3, 4, -5, ...</tt>.
-     * The corresponding output for the first segment will be <tt>INT_MIN, 0, 0, 2, 2, 4, ..., 508, 510</tt>.
-     * The output for the second segment will be <tt>510, 512, 512, 514, 514, 516, ..., 1020, 1022</tt>.
-     *
-     * \tparam ITEMS_PER_THREAD         <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam ScanOp                   <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     * \tparam BlockPrefixCallbackOp    <b>[inferred]</b> Call-back functor type having member <tt>T operator()(T block_aggregate)</tt>
-     */
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        ScanOp,
-        typename        BlockPrefixCallbackOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T                       (&input)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input items
-        T                       (&output)[ITEMS_PER_THREAD],    ///< [out] Calling thread's output items (may be aliased to \p input)
-        ScanOp                  scan_op,                        ///< [in] Binary scan functor
-        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a block-wide prefix to be applied to the logical input sequence.
-    {
-        // Reduce consecutive thread items in registers
-        T thread_prefix = internal::ThreadReduce(input, scan_op);
-
-        // Exclusive thread block-scan
-        ExclusiveScan(thread_prefix, thread_prefix, scan_op, block_prefix_callback_op);
-
-        // Exclusive scan in registers with prefix as seed
-        internal::ThreadScanExclusive(input, output, scan_op, thread_prefix);
-    }
-
-
-    //@}  end member group
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document no-initial-value scans
-
-    /******************************************************************//**
-     * \name Exclusive prefix scan operations (no initial value, single datum per thread)
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
-     *
-     * \par
-     * - Supports non-commutative scan operators.
-     * - \rowmajor
-     * - \smemreuse
-     *
-     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op)                        ///< [in] Binary scan functor
-    {
-        InternalBlockScan(temp_storage).ExclusiveScan(input, output, scan_op);
-    }
-
-
-    /**
-     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
-     *
-     * \par
-     * - Supports non-commutative scan operators.
-     * - \rowmajor
-     * - \smemreuse
-     *
-     * \tparam ScanOp   <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op,                        ///< [in] Binary scan functor
-        T               &block_aggregate)               ///< [out] block-wide aggregate reduction of input items
-    {
-        InternalBlockScan(temp_storage).ExclusiveScan(input, output, scan_op, block_aggregate);
-    }
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Exclusive prefix scan operations (no initial value, multiple data per thread)
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes an array of consecutive input elements.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
-     *
-     * \par
-     * - Supports non-commutative scan operators.
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T                 (&input)[ITEMS_PER_THREAD],   ///< [in] Calling thread's input items
-        T                 (&output)[ITEMS_PER_THREAD],  ///< [out] Calling thread's output items (may be aliased to \p input)
-        ScanOp            scan_op)                      ///< [in] Binary scan functor
-    {
-        // Reduce consecutive thread items in registers
-        T thread_partial = internal::ThreadReduce(input, scan_op);
-
-        // Exclusive thread block-scan
-        ExclusiveScan(thread_partial, thread_partial, scan_op);
-
-        // Exclusive scan in registers with prefix
-        internal::ThreadScanExclusive(input, output, scan_op, thread_partial, (linear_tid != 0));
-    }
-
-
-    /**
-     * \brief Computes an exclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes an array of consecutive input elements.  Also provides every thread with the block-wide \p block_aggregate of all inputs.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
-     *
-     * \par
-     * - Supports non-commutative scan operators.
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               (&input)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input items
-        T               (&output)[ITEMS_PER_THREAD],    ///< [out] Calling thread's output items (may be aliased to \p input)
-        ScanOp          scan_op,                        ///< [in] Binary scan functor
-        T               &block_aggregate)               ///< [out] block-wide aggregate reduction of input items
-    {
-        // Reduce consecutive thread items in registers
-        T thread_partial = internal::ThreadReduce(input, scan_op);
-
-        // Exclusive thread block-scan
-        ExclusiveScan(thread_partial, thread_partial, scan_op, block_aggregate);
-
-        // Exclusive scan in registers with prefix
-        internal::ThreadScanExclusive(input, output, scan_op, thread_partial, (linear_tid != 0));
-    }
-
-
-    //@}  end member group
-#endif // DOXYGEN_SHOULD_SKIP_THIS  // Do not document no-initial-value scans
-
-    /******************************************************************//**
-     * \name Inclusive prefix sum operations
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Computes an inclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes one input element.
-     *
-     * \par
-     * - \rowmajor
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates an inclusive prefix sum of 128 integer items that
-     * are partitioned across 128 threads.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads on type int
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Obtain input item for each thread
-     *     int thread_data;
-     *     ...
-     *
-     *     // Collectively compute the block-wide inclusive prefix sum
-     *     BlockScan(temp_storage).InclusiveSum(thread_data, thread_data);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>1, 1, ..., 1</tt>.  The
-     * corresponding output \p thread_data in those threads will be <tt>1, 2, ..., 128</tt>.
-     *
-     */
-    __device__ __forceinline__ void InclusiveSum(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &output)                        ///< [out] Calling thread's output item (may be aliased to \p input)
-    {
-        InclusiveScan(input, output, cub::Sum());
-    }
-
-
-    /**
-     * \brief Computes an inclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-     *
-     * \par
-     * - \rowmajor
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates an inclusive prefix sum of 128 integer items that
-     * are partitioned across 128 threads.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads on type int
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Obtain input item for each thread
-     *     int thread_data;
-     *     ...
-     *
-     *     // Collectively compute the block-wide inclusive prefix sum
-     *     int block_aggregate;
-     *     BlockScan(temp_storage).InclusiveSum(thread_data, thread_data, block_aggregate);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>1, 1, ..., 1</tt>.  The
-     * corresponding output \p thread_data in those threads will be <tt>1, 2, ..., 128</tt>.
-     * Furthermore the value \p 128 will be stored in \p block_aggregate for all threads.
-     *
-     */
-    __device__ __forceinline__ void InclusiveSum(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
-        T               &block_aggregate)               ///< [out] block-wide aggregate reduction of input items
-    {
-        InclusiveScan(input, output, cub::Sum(), block_aggregate);
-    }
-
-
-
-    /**
-     * \brief Computes an inclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes one input element.  Instead of using 0 as the block-wide prefix, the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-     *
-     * \par
-     * - The \p block_prefix_callback_op functor must implement a member function <tt>T operator()(T block_aggregate)</tt>.
-     *   The functor's input parameter \p block_aggregate is the same value also returned by the scan operation.
-     *   The functor will be invoked by the first warp of threads in the block, however only the return value from
-     *   <em>lane</em><sub>0</sub> is applied as the block-wide prefix.  Can be stateful.
-     * - \rowmajor
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a single thread block that progressively
-     * computes an inclusive prefix sum over multiple "tiles" of input using a
-     * prefix functor to maintain a running total between block-wide scans.  Each tile consists
-     * of 128 integer items that are partitioned across 128 threads.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * // A stateful callback functor that maintains a running prefix to be applied
-     * // during consecutive scan operations.
-     * struct BlockPrefixCallbackOp
-     * {
-     *     // Running prefix
-     *     int running_total;
-     *
-     *     // Constructor
-     *     __device__ BlockPrefixCallbackOp(int running_total) : running_total(running_total) {}
-     *
-     *     // Callback operator to be entered by the first warp of threads in the block.
-     *     // Thread-0 is responsible for returning a value for seeding the block-wide scan.
-     *     __device__ int operator()(int block_aggregate)
-     *     {
-     *         int old_prefix = running_total;
-     *         running_total += block_aggregate;
-     *         return old_prefix;
-     *     }
-     * };
-     *
-     * __global__ void ExampleKernel(int *d_data, int num_items, ...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Initialize running total
-     *     BlockPrefixCallbackOp prefix_op(0);
-     *
-     *     // Have the block iterate over segments of items
-     *     for (int block_offset = 0; block_offset < num_items; block_offset += 128)
-     *     {
-     *         // Load a segment of consecutive items that are blocked across threads
-     *         int thread_data = d_data[block_offset];
-     *
-     *         // Collectively compute the block-wide inclusive prefix sum
-     *         BlockScan(temp_storage).InclusiveSum(
-     *             thread_data, thread_data, prefix_op);
-     *         CTA_SYNC();
-     *
-     *         // Store scanned items to output segment
-     *         d_data[block_offset] = thread_data;
-     *     }
-     * \endcode
-     * \par
-     * Suppose the input \p d_data is <tt>1, 1, 1, 1, 1, 1, 1, 1, ...</tt>.
-     * The corresponding output for the first segment will be <tt>1, 2, ..., 128</tt>.
-     * The output for the second segment will be <tt>129, 130, ..., 256</tt>.
-     *
-     * \tparam BlockPrefixCallbackOp          <b>[inferred]</b> Call-back functor type having member <tt>T operator()(T block_aggregate)</tt>
-     */
-    template <typename BlockPrefixCallbackOp>
-    __device__ __forceinline__ void InclusiveSum(
-        T                       input,                          ///< [in] Calling thread's input item
-        T                       &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
-        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a block-wide prefix to be applied to the logical input sequence.
-    {
-        InclusiveScan(input, output, cub::Sum(), block_prefix_callback_op);
-    }
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Inclusive prefix sum operations (multiple data per thread)
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Computes an inclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes an array of consecutive input elements.
-     *
-     * \par
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates an inclusive prefix sum of 512 integer items that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads on type int
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Collectively compute the block-wide inclusive prefix sum
-     *     BlockScan(temp_storage).InclusiveSum(thread_data, thread_data);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{ [1,1,1,1], [1,1,1,1], ..., [1,1,1,1] }</tt>.  The
-     * corresponding output \p thread_data in those threads will be <tt>{ [1,2,3,4], [5,6,7,8], ..., [509,510,511,512] }</tt>.
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     */
-    template <int ITEMS_PER_THREAD>
-    __device__ __forceinline__ void InclusiveSum(
-        T               (&input)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input items
-        T               (&output)[ITEMS_PER_THREAD])    ///< [out] Calling thread's output items (may be aliased to \p input)
-    {
-        if (ITEMS_PER_THREAD == 1)
-        {
-            InclusiveSum(input[0], output[0]);
-        }
-        else
-        {
-            // Reduce consecutive thread items in registers
-            Sum scan_op;
-            T thread_prefix = internal::ThreadReduce(input, scan_op);
-
-            // Exclusive thread block-scan
-            ExclusiveSum(thread_prefix, thread_prefix);
-
-            // Inclusive scan in registers with prefix as seed
-            internal::ThreadScanInclusive(input, output, scan_op, thread_prefix, (linear_tid != 0));
-        }
-    }
-
-
-    /**
-     * \brief Computes an inclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes an array of consecutive input elements.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-     *
-     * \par
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates an inclusive prefix sum of 512 integer items that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads on type int
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Collectively compute the block-wide inclusive prefix sum
-     *     int block_aggregate;
-     *     BlockScan(temp_storage).InclusiveSum(thread_data, thread_data, block_aggregate);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is
-     * <tt>{ [1,1,1,1], [1,1,1,1], ..., [1,1,1,1] }</tt>.  The
-     * corresponding output \p thread_data in those threads will be
-     * <tt>{ [1,2,3,4], [5,6,7,8], ..., [509,510,511,512] }</tt>.
-     * Furthermore the value \p 512 will be stored in \p block_aggregate for all threads.
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <int ITEMS_PER_THREAD>
-    __device__ __forceinline__ void InclusiveSum(
-        T               (&input)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input items
-        T               (&output)[ITEMS_PER_THREAD],    ///< [out] Calling thread's output items (may be aliased to \p input)
-        T               &block_aggregate)               ///< [out] block-wide aggregate reduction of input items
-    {
-        if (ITEMS_PER_THREAD == 1)
-        {
-            InclusiveSum(input[0], output[0], block_aggregate);
-        }
-        else
-        {
-            // Reduce consecutive thread items in registers
-            Sum scan_op;
-            T thread_prefix = internal::ThreadReduce(input, scan_op);
-
-            // Exclusive thread block-scan
-            ExclusiveSum(thread_prefix, thread_prefix, block_aggregate);
-
-            // Inclusive scan in registers with prefix as seed
-            internal::ThreadScanInclusive(input, output, scan_op, thread_prefix, (linear_tid != 0));
-        }
-    }
-
-
-    /**
-     * \brief Computes an inclusive block-wide prefix scan using addition (+) as the scan operator.  Each thread contributes an array of consecutive input elements.  Instead of using 0 as the block-wide prefix, the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-     *
-     * \par
-     * - The \p block_prefix_callback_op functor must implement a member function <tt>T operator()(T block_aggregate)</tt>.
-     *   The functor's input parameter \p block_aggregate is the same value also returned by the scan operation.
-     *   The functor will be invoked by the first warp of threads in the block, however only the return value from
-     *   <em>lane</em><sub>0</sub> is applied as the block-wide prefix.  Can be stateful.
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a single thread block that progressively
-     * computes an inclusive prefix sum over multiple "tiles" of input using a
-     * prefix functor to maintain a running total between block-wide scans.  Each tile consists
-     * of 512 integer items that are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3)
-     * across 128 threads where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * // A stateful callback functor that maintains a running prefix to be applied
-     * // during consecutive scan operations.
-     * struct BlockPrefixCallbackOp
-     * {
-     *     // Running prefix
-     *     int running_total;
-     *
-     *     // Constructor
-     *     __device__ BlockPrefixCallbackOp(int running_total) : running_total(running_total) {}
-     *
-     *     // Callback operator to be entered by the first warp of threads in the block.
-     *     // Thread-0 is responsible for returning a value for seeding the block-wide scan.
-     *     __device__ int operator()(int block_aggregate)
-     *     {
-     *         int old_prefix = running_total;
-     *         running_total += block_aggregate;
-     *         return old_prefix;
-     *     }
-     * };
-     *
-     * __global__ void ExampleKernel(int *d_data, int num_items, ...)
-     * {
-     *     // Specialize BlockLoad, BlockStore, and BlockScan for a 1D block of 128 threads, 4 ints per thread
-     *     typedef cub::BlockLoad<int*, 128, 4, BLOCK_LOAD_TRANSPOSE>   BlockLoad;
-     *     typedef cub::BlockStore<int, 128, 4, BLOCK_STORE_TRANSPOSE>  BlockStore;
-     *     typedef cub::BlockScan<int, 128>                             BlockScan;
-     *
-     *     // Allocate aliased shared memory for BlockLoad, BlockStore, and BlockScan
-     *     __shared__ union {
-     *         typename BlockLoad::TempStorage     load;
-     *         typename BlockScan::TempStorage     scan;
-     *         typename BlockStore::TempStorage    store;
-     *     } temp_storage;
-     *
-     *     // Initialize running total
-     *     BlockPrefixCallbackOp prefix_op(0);
-     *
-     *     // Have the block iterate over segments of items
-     *     for (int block_offset = 0; block_offset < num_items; block_offset += 128 * 4)
-     *     {
-     *         // Load a segment of consecutive items that are blocked across threads
-     *         int thread_data[4];
-     *         BlockLoad(temp_storage.load).Load(d_data + block_offset, thread_data);
-     *         CTA_SYNC();
-     *
-     *         // Collectively compute the block-wide inclusive prefix sum
-     *         BlockScan(temp_storage.scan).IncluisveSum(
-     *             thread_data, thread_data, prefix_op);
-     *         CTA_SYNC();
-     *
-     *         // Store scanned items to output segment
-     *         BlockStore(temp_storage.store).Store(d_data + block_offset, thread_data);
-     *         CTA_SYNC();
-     *     }
-     * \endcode
-     * \par
-     * Suppose the input \p d_data is <tt>1, 1, 1, 1, 1, 1, 1, 1, ...</tt>.
-     * The corresponding output for the first segment will be <tt>1, 2, 3, 4, ..., 511, 512</tt>.
-     * The output for the second segment will be <tt>513, 514, 515, 516, ..., 1023, 1024</tt>.
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam BlockPrefixCallbackOp        <b>[inferred]</b> Call-back functor type having member <tt>T operator()(T block_aggregate)</tt>
-     */
-    template <
-        int ITEMS_PER_THREAD,
-        typename BlockPrefixCallbackOp>
-    __device__ __forceinline__ void InclusiveSum(
-        T                       (&input)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input items
-        T                       (&output)[ITEMS_PER_THREAD],    ///< [out] Calling thread's output items (may be aliased to \p input)
-        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a block-wide prefix to be applied to the logical input sequence.
-    {
-        if (ITEMS_PER_THREAD == 1)
-        {
-            InclusiveSum(input[0], output[0], block_prefix_callback_op);
-        }
-        else
-        {
-            // Reduce consecutive thread items in registers
-            Sum scan_op;
-            T thread_prefix = internal::ThreadReduce(input, scan_op);
-
-            // Exclusive thread block-scan
-            ExclusiveSum(thread_prefix, thread_prefix, block_prefix_callback_op);
-
-            // Inclusive scan in registers with prefix as seed
-            internal::ThreadScanInclusive(input, output, scan_op, thread_prefix);
-        }
-    }
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Inclusive prefix scan operations
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Computes an inclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
-     *
-     * \par
-     * - Supports non-commutative scan operators.
-     * - \rowmajor
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates an inclusive prefix max scan of 128 integer items that
-     * are partitioned across 128 threads.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads on type int
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Obtain input item for each thread
-     *     int thread_data;
-     *     ...
-     *
-     *     // Collectively compute the block-wide inclusive prefix max scan
-     *     BlockScan(temp_storage).InclusiveScan(thread_data, thread_data, cub::Max());
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>0, -1, 2, -3, ..., 126, -127</tt>.  The
-     * corresponding output \p thread_data in those threads will be <tt>0, 0, 2, 2, ..., 126, 126</tt>.
-     *
-     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <typename ScanOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op)                        ///< [in] Binary scan functor 
-    {
-        InternalBlockScan(temp_storage).InclusiveScan(input, output, scan_op);
-    }
-
-
-    /**
-     * \brief Computes an inclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-     *
-     * \par
-     * - Supports non-commutative scan operators.
-     * - \rowmajor
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates an inclusive prefix max scan of 128 integer items that
-     * are partitioned across 128 threads.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads on type int
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Obtain input item for each thread
-     *     int thread_data;
-     *     ...
-     *
-     *     // Collectively compute the block-wide inclusive prefix max scan
-     *     int block_aggregate;
-     *     BlockScan(temp_storage).InclusiveScan(thread_data, thread_data, cub::Max(), block_aggregate);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>0, -1, 2, -3, ..., 126, -127</tt>.  The
-     * corresponding output \p thread_data in those threads will be <tt>0, 0, 2, 2, ..., 126, 126</tt>.
-     * Furthermore the value \p 126 will be stored in \p block_aggregate for all threads.
-     *
-     * \tparam ScanOp   <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <typename ScanOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op,                        ///< [in] Binary scan functor 
-        T               &block_aggregate)               ///< [out] block-wide aggregate reduction of input items
-    {
-        InternalBlockScan(temp_storage).InclusiveScan(input, output, scan_op, block_aggregate);
-    }
-
-
-    /**
-     * \brief Computes an inclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-     *
-     * \par
-     * - The \p block_prefix_callback_op functor must implement a member function <tt>T operator()(T block_aggregate)</tt>.
-     *   The functor's input parameter \p block_aggregate is the same value also returned by the scan operation.
-     *   The functor will be invoked by the first warp of threads in the block, however only the return value from
-     *   <em>lane</em><sub>0</sub> is applied as the block-wide prefix.  Can be stateful.
-     * - Supports non-commutative scan operators.
-     * - \rowmajor
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a single thread block that progressively
-     * computes an inclusive prefix max scan over multiple "tiles" of input using a
-     * prefix functor to maintain a running total between block-wide scans.  Each tile consists
-     * of 128 integer items that are partitioned across 128 threads.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * // A stateful callback functor that maintains a running prefix to be applied
-     * // during consecutive scan operations.
-     * struct BlockPrefixCallbackOp
-     * {
-     *     // Running prefix
-     *     int running_total;
-     *
-     *     // Constructor
-     *     __device__ BlockPrefixCallbackOp(int running_total) : running_total(running_total) {}
-     *
-     *     // Callback operator to be entered by the first warp of threads in the block.
-     *     // Thread-0 is responsible for returning a value for seeding the block-wide scan.
-     *     __device__ int operator()(int block_aggregate)
-     *     {
-     *         int old_prefix = running_total;
-     *         running_total = (block_aggregate > old_prefix) ? block_aggregate : old_prefix;
-     *         return old_prefix;
-     *     }
-     * };
-     *
-     * __global__ void ExampleKernel(int *d_data, int num_items, ...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Initialize running total
-     *     BlockPrefixCallbackOp prefix_op(INT_MIN);
-     *
-     *     // Have the block iterate over segments of items
-     *     for (int block_offset = 0; block_offset < num_items; block_offset += 128)
-     *     {
-     *         // Load a segment of consecutive items that are blocked across threads
-     *         int thread_data = d_data[block_offset];
-     *
-     *         // Collectively compute the block-wide inclusive prefix max scan
-     *         BlockScan(temp_storage).InclusiveScan(
-     *             thread_data, thread_data, cub::Max(), prefix_op);
-     *         CTA_SYNC();
-     *
-     *         // Store scanned items to output segment
-     *         d_data[block_offset] = thread_data;
-     *     }
-     * \endcode
-     * \par
-     * Suppose the input \p d_data is <tt>0, -1, 2, -3, 4, -5, ...</tt>.
-     * The corresponding output for the first segment will be <tt>0, 0, 2, 2, ..., 126, 126</tt>.
-     * The output for the second segment will be <tt>128, 128, 130, 130, ..., 254, 254</tt>.
-     *
-     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     * \tparam BlockPrefixCallbackOp        <b>[inferred]</b> Call-back functor type having member <tt>T operator()(T block_aggregate)</tt>
-     */
-    template <
-        typename ScanOp,
-        typename BlockPrefixCallbackOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T                       input,                          ///< [in] Calling thread's input item
-        T                       &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp                  scan_op,                        ///< [in] Binary scan functor 
-        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a block-wide prefix to be applied to the logical input sequence.
-    {
-        InternalBlockScan(temp_storage).InclusiveScan(input, output, scan_op, block_prefix_callback_op);
-    }
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Inclusive prefix scan operations (multiple data per thread)
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Computes an inclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes an array of consecutive input elements.
-     *
-     * \par
-     * - Supports non-commutative scan operators.
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates an inclusive prefix max scan of 512 integer items that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads on type int
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Collectively compute the block-wide inclusive prefix max scan
-     *     BlockScan(temp_storage).InclusiveScan(thread_data, thread_data, cub::Max());
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{ [0,-1,2,-3], [4,-5,6,-7], ..., [508,-509,510,-511] }</tt>.  The
-     * corresponding output \p thread_data in those threads will be <tt>{ [0,0,2,2], [4,4,6,6], ..., [508,508,510,510] }</tt>.
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        ScanOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T               (&input)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input items
-        T               (&output)[ITEMS_PER_THREAD],    ///< [out] Calling thread's output items (may be aliased to \p input)
-        ScanOp          scan_op)                        ///< [in] Binary scan functor 
-    {
-        if (ITEMS_PER_THREAD == 1)
-        {
-            InclusiveScan(input[0], output[0], scan_op);
-        }
-        else
-        {
-            // Reduce consecutive thread items in registers
-            T thread_prefix = internal::ThreadReduce(input, scan_op);
-
-            // Exclusive thread block-scan
-            ExclusiveScan(thread_prefix, thread_prefix, scan_op);
-
-            // Inclusive scan in registers with prefix as seed (first thread does not seed)
-            internal::ThreadScanInclusive(input, output, scan_op, thread_prefix, (linear_tid != 0));
-        }
-    }
-
-
-    /**
-     * \brief Computes an inclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes an array of consecutive input elements.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-     *
-     * \par
-     * - Supports non-commutative scan operators.
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates an inclusive prefix max scan of 512 integer items that
-     * are partitioned in a [<em>blocked arrangement</em>](index.html#sec5sec3) across 128 threads
-     * where each thread owns 4 consecutive items.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize BlockScan for a 1D block of 128 threads on type int
-     *     typedef cub::BlockScan<int, 128> BlockScan;
-     *
-     *     // Allocate shared memory for BlockScan
-     *     __shared__ typename BlockScan::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Collectively compute the block-wide inclusive prefix max scan
-     *     int block_aggregate;
-     *     BlockScan(temp_storage).InclusiveScan(thread_data, thread_data, cub::Max(), block_aggregate);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is
-     * <tt>{ [0,-1,2,-3], [4,-5,6,-7], ..., [508,-509,510,-511] }</tt>.
-     * The corresponding output \p thread_data in those threads will be
-     * <tt>{ [0,0,2,2], [4,4,6,6], ..., [508,508,510,510] }</tt>.
-     * Furthermore the value \p 510 will be stored in \p block_aggregate for all threads.
-     *
-     * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam ScanOp               <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <
-        int             ITEMS_PER_THREAD,
-        typename         ScanOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T               (&input)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input items
-        T               (&output)[ITEMS_PER_THREAD],    ///< [out] Calling thread's output items (may be aliased to \p input)
-        ScanOp          scan_op,                        ///< [in] Binary scan functor 
-        T               &block_aggregate)               ///< [out] block-wide aggregate reduction of input items
-    {
-        if (ITEMS_PER_THREAD == 1)
-        {
-            InclusiveScan(input[0], output[0], scan_op, block_aggregate);
-        }
-        else
-        {
-            // Reduce consecutive thread items in registers
-            T thread_prefix = internal::ThreadReduce(input, scan_op);
-
-            // Exclusive thread block-scan (with no initial value)
-            ExclusiveScan(thread_prefix, thread_prefix, scan_op, block_aggregate);
-
-            // Inclusive scan in registers with prefix as seed (first thread does not seed)
-            internal::ThreadScanInclusive(input, output, scan_op, thread_prefix, (linear_tid != 0));
-        }
-    }
-
-
-    /**
-     * \brief Computes an inclusive block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes an array of consecutive input elements.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-     *
-     * \par
-     * - The \p block_prefix_callback_op functor must implement a member function <tt>T operator()(T block_aggregate)</tt>.
-     *   The functor's input parameter \p block_aggregate is the same value also returned by the scan operation.
-     *   The functor will be invoked by the first warp of threads in the block, however only the return value from
-     *   <em>lane</em><sub>0</sub> is applied as the block-wide prefix.  Can be stateful.
-     * - Supports non-commutative scan operators.
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a single thread block that progressively
-     * computes an inclusive prefix max scan over multiple "tiles" of input using a
-     * prefix functor to maintain a running total between block-wide scans.  Each tile consists
-     * of 128 integer items that are partitioned across 128 threads.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_scan.cuh>
-     *
-     * // A stateful callback functor that maintains a running prefix to be applied
-     * // during consecutive scan operations.
-     * struct BlockPrefixCallbackOp
-     * {
-     *     // Running prefix
-     *     int running_total;
-     *
-     *     // Constructor
-     *     __device__ BlockPrefixCallbackOp(int running_total) : running_total(running_total) {}
-     *
-     *     // Callback operator to be entered by the first warp of threads in the block.
-     *     // Thread-0 is responsible for returning a value for seeding the block-wide scan.
-     *     __device__ int operator()(int block_aggregate)
-     *     {
-     *         int old_prefix = running_total;
-     *         running_total = (block_aggregate > old_prefix) ? block_aggregate : old_prefix;
-     *         return old_prefix;
-     *     }
-     * };
-     *
-     * __global__ void ExampleKernel(int *d_data, int num_items, ...)
-     * {
-     *     // Specialize BlockLoad, BlockStore, and BlockScan for a 1D block of 128 threads, 4 ints per thread
-     *     typedef cub::BlockLoad<int*, 128, 4, BLOCK_LOAD_TRANSPOSE>   BlockLoad;
-     *     typedef cub::BlockStore<int, 128, 4, BLOCK_STORE_TRANSPOSE>  BlockStore;
-     *     typedef cub::BlockScan<int, 128>                             BlockScan;
-     *
-     *     // Allocate aliased shared memory for BlockLoad, BlockStore, and BlockScan
-     *     __shared__ union {
-     *         typename BlockLoad::TempStorage     load;
-     *         typename BlockScan::TempStorage     scan;
-     *         typename BlockStore::TempStorage    store;
-     *     } temp_storage;
-     *
-     *     // Initialize running total
-     *     BlockPrefixCallbackOp prefix_op(0);
-     *
-     *     // Have the block iterate over segments of items
-     *     for (int block_offset = 0; block_offset < num_items; block_offset += 128 * 4)
-     *     {
-     *         // Load a segment of consecutive items that are blocked across threads
-     *         int thread_data[4];
-     *         BlockLoad(temp_storage.load).Load(d_data + block_offset, thread_data);
-     *         CTA_SYNC();
-     *
-     *         // Collectively compute the block-wide inclusive prefix max scan
-     *         BlockScan(temp_storage.scan).InclusiveScan(
-     *             thread_data, thread_data, cub::Max(), prefix_op);
-     *         CTA_SYNC();
-     *
-     *         // Store scanned items to output segment
-     *         BlockStore(temp_storage.store).Store(d_data + block_offset, thread_data);
-     *         CTA_SYNC();
-     *     }
-     * \endcode
-     * \par
-     * Suppose the input \p d_data is <tt>0, -1, 2, -3, 4, -5, ...</tt>.
-     * The corresponding output for the first segment will be <tt>0, 0, 2, 2, 4, 4, ..., 510, 510</tt>.
-     * The output for the second segment will be <tt>512, 512, 514, 514, 516, 516, ..., 1022, 1022</tt>.
-     *
-     * \tparam ITEMS_PER_THREAD         <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
-     * \tparam ScanOp                   <b>[inferred]</b> Binary scan functor  type having member <tt>T operator()(const T &a, const T &b)</tt>
-     * \tparam BlockPrefixCallbackOp    <b>[inferred]</b> Call-back functor type having member <tt>T operator()(T block_aggregate)</tt>
-     */
-    template <
-        int             ITEMS_PER_THREAD,
-        typename        ScanOp,
-        typename        BlockPrefixCallbackOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T                       (&input)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input items
-        T                       (&output)[ITEMS_PER_THREAD],    ///< [out] Calling thread's output items (may be aliased to \p input)
-        ScanOp                  scan_op,                        ///< [in] Binary scan functor 
-        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a block-wide prefix to be applied to the logical input sequence.
-    {
-        if (ITEMS_PER_THREAD == 1)
-        {
-            InclusiveScan(input[0], output[0], scan_op, block_prefix_callback_op);
-        }
-        else
-        {
-            // Reduce consecutive thread items in registers
-            T thread_prefix = internal::ThreadReduce(input, scan_op);
-
-            // Exclusive thread block-scan
-            ExclusiveScan(thread_prefix, thread_prefix, scan_op, block_prefix_callback_op);
-
-            // Inclusive scan in registers with prefix as seed
-            internal::ThreadScanInclusive(input, output, scan_op, thread_prefix);
-        }
-    }
-
-    //@}  end member group
-
-
-};
-
-/**
- * \example example_block_scan.cu
- */
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/block/block_shuffle.cuh b/pfsimulator/third_party/cub/block/block_shuffle.cuh
deleted file mode 100644
index a0cc71d22..000000000
--- a/pfsimulator/third_party/cub/block/block_shuffle.cuh
+++ /dev/null
@@ -1,305 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * The cub::BlockShuffle class provides [<em>collective</em>](index.html#sec0) methods for shuffling data partitioned across a CUDA thread block.
- */
-
-#pragma once
-
-#include "../util_arch.cuh"
-#include "../util_ptx.cuh"
-#include "../util_macro.cuh"
-#include "../util_type.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \brief The BlockShuffle class provides [<em>collective</em>](index.html#sec0) methods for shuffling data partitioned across a CUDA thread block.
- * \ingroup BlockModule
- *
- * \tparam T                    The data type to be exchanged.
- * \tparam BLOCK_DIM_X          The thread block length in threads along the X dimension
- * \tparam BLOCK_DIM_Y          <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
- * \tparam BLOCK_DIM_Z          <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
- * \tparam PTX_ARCH             <b>[optional]</b> \ptxversion
- *
- * \par Overview
- * It is commonplace for blocks of threads to rearrange data items between
- * threads.  The BlockShuffle abstraction allows threads to efficiently shift items
- * either (a) up to their successor or (b) down to their predecessor.
- *
- */
-template <
-    typename            T,
-    int                 BLOCK_DIM_X,
-    int                 BLOCK_DIM_Y         = 1,
-    int                 BLOCK_DIM_Z         = 1,
-    int                 PTX_ARCH            = CUB_PTX_ARCH>
-class BlockShuffle
-{
-private:
-
-    /******************************************************************************
-     * Constants
-     ******************************************************************************/
-
-    enum
-    {
-        BLOCK_THREADS               = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-
-        LOG_WARP_THREADS            = CUB_LOG_WARP_THREADS(PTX_ARCH),
-        WARP_THREADS                = 1 << LOG_WARP_THREADS,
-        WARPS                       = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS,
-    };
-
-    /******************************************************************************
-     * Type definitions
-     ******************************************************************************/
-
-    /// Shared memory storage layout type (last element from each thread's input)
-    struct _TempStorage
-    {
-        T prev[BLOCK_THREADS];
-        T next[BLOCK_THREADS];
-    };
-
-
-public:
-
-    /// \smemstorage{BlockShuffle}
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-private:
-
-
-    /******************************************************************************
-     * Thread fields
-     ******************************************************************************/
-
-    /// Shared storage reference
-    _TempStorage &temp_storage;
-
-    /// Linear thread-id
-    unsigned int linear_tid;
-
-
-    /******************************************************************************
-     * Utility methods
-     ******************************************************************************/
-
-    /// Internal storage allocator
-    __device__ __forceinline__ _TempStorage& PrivateStorage()
-    {
-        __shared__ _TempStorage private_storage;
-        return private_storage;
-    }
-
-
-public:
-
-    /******************************************************************//**
-     * \name Collective constructors
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
-     */
-    __device__ __forceinline__ BlockShuffle()
-    :
-        temp_storage(PrivateStorage()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    /**
-     * \brief Collective constructor using the specified memory allocation as temporary storage.
-     */
-    __device__ __forceinline__ BlockShuffle(
-        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Shuffle movement
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Each <em>thread<sub>i</sub></em> obtains the \p input provided by <em>thread</em><sub><em>i</em>+<tt>distance</tt></sub>. The offset \p distance may be negative.
-     *
-     * \par
-     * - \smemreuse
-     */
-    __device__ __forceinline__ void Offset(
-        T   input,                  ///< [in] The input item from the calling thread (<em>thread<sub>i</sub></em>)
-        T&  output,                 ///< [out] The \p input item from the successor (or predecessor) thread <em>thread</em><sub><em>i</em>+<tt>distance</tt></sub> (may be aliased to \p input).  This value is only updated for for <em>thread<sub>i</sub></em> when 0 <= (<em>i</em> + \p distance) < <tt>BLOCK_THREADS-1</tt>
-        int distance = 1)           ///< [in] Offset distance (may be negative)
-    {
-        temp_storage[linear_tid].prev = input;
-
-        CTA_SYNC();
-
-        if ((linear_tid + distance >= 0) && (linear_tid + distance < BLOCK_THREADS))
-            output = temp_storage[linear_tid + distance].prev;
-    }
-
-
-    /**
-     * \brief Each <em>thread<sub>i</sub></em> obtains the \p input provided by <em>thread</em><sub><em>i</em>+<tt>distance</tt></sub>.
-     *
-     * \par
-     * - \smemreuse
-     */
-    __device__ __forceinline__ void Rotate(
-        T   input,                  ///< [in] The calling thread's input item
-        T&  output,                 ///< [out] The \p input item from thread <em>thread</em><sub>(<em>i</em>+<tt>distance></tt>)%<tt><BLOCK_THREADS></tt></sub> (may be aliased to \p input).  This value is not updated for <em>thread</em><sub>BLOCK_THREADS-1</sub>
-        unsigned int distance = 1)  ///< [in] Offset distance (0 < \p distance < <tt>BLOCK_THREADS</tt>)
-    {
-        temp_storage[linear_tid].prev = input;
-
-        CTA_SYNC();
-
-        unsigned int offset = threadIdx.x + distance;
-        if (offset >= BLOCK_THREADS)
-            offset -= BLOCK_THREADS;
-
-        output = temp_storage[offset].prev;
-    }
-
-
-    /**
-     * \brief The thread block rotates its [<em>blocked arrangement</em>](index.html#sec5sec3) of \p input items, shifting it up by one item
-     *
-     * \par
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     */
-    template <int ITEMS_PER_THREAD>
-    __device__ __forceinline__ void Up(
-        T (&input)[ITEMS_PER_THREAD],   ///< [in] The calling thread's input items
-        T (&prev)[ITEMS_PER_THREAD])    ///< [out] The corresponding predecessor items (may be aliased to \p input).  The item \p prev[0] is not updated for <em>thread</em><sub>0</sub>.
-    {
-        temp_storage[linear_tid].prev = input[ITEMS_PER_THREAD - 1];
-
-        CTA_SYNC();
-
-        #pragma unroll
-        for (int ITEM = ITEMS_PER_THREAD - 1; ITEM > 0; --ITEM)
-            prev[ITEM] = input[ITEM - 1];
-
-
-        if (linear_tid > 0)
-            prev[0] = temp_storage[linear_tid - 1].prev;
-    }
-
-
-    /**
-     * \brief The thread block rotates its [<em>blocked arrangement</em>](index.html#sec5sec3) of \p input items, shifting it up by one item.  All threads receive the \p input provided by <em>thread</em><sub><tt>BLOCK_THREADS-1</tt></sub>.
-     *
-     * \par
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     */
-    template <int ITEMS_PER_THREAD>
-    __device__ __forceinline__ void Up(
-        T (&input)[ITEMS_PER_THREAD],   ///< [in] The calling thread's input items
-        T (&prev)[ITEMS_PER_THREAD],    ///< [out] The corresponding predecessor items (may be aliased to \p input).  The item \p prev[0] is not updated for <em>thread</em><sub>0</sub>.
-        T &block_suffix)                ///< [out] The item \p input[ITEMS_PER_THREAD-1] from <em>thread</em><sub><tt>BLOCK_THREADS-1</tt></sub>, provided to all threads
-    {
-        Up(input, prev);
-        block_suffix = temp_storage[BLOCK_THREADS - 1].prev;
-    }
-
-
-    /**
-     * \brief The thread block rotates its [<em>blocked arrangement</em>](index.html#sec5sec3) of \p input items, shifting it down by one item
-     *
-     * \par
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     */
-    template <int ITEMS_PER_THREAD>
-    __device__ __forceinline__ void Down(
-        T (&input)[ITEMS_PER_THREAD],   ///< [in] The calling thread's input items
-        T (&prev)[ITEMS_PER_THREAD])    ///< [out] The corresponding predecessor items (may be aliased to \p input).  The value \p prev[0] is not updated for <em>thread</em><sub>BLOCK_THREADS-1</sub>.
-    {
-        temp_storage[linear_tid].prev = input[ITEMS_PER_THREAD - 1];
-
-        CTA_SYNC();
-
-        #pragma unroll
-        for (int ITEM = ITEMS_PER_THREAD - 1; ITEM > 0; --ITEM)
-            prev[ITEM] = input[ITEM - 1];
-
-        if (linear_tid > 0)
-            prev[0] = temp_storage[linear_tid - 1].prev;
-    }
-
-
-    /**
-     * \brief The thread block rotates its [<em>blocked arrangement</em>](index.html#sec5sec3) of input items, shifting it down by one item.  All threads receive \p input[0] provided by <em>thread</em><sub><tt>0</tt></sub>.
-     *
-     * \par
-     * - \blocked
-     * - \granularity
-     * - \smemreuse
-     */
-    template <int ITEMS_PER_THREAD>
-    __device__ __forceinline__ void Down(
-        T (&input)[ITEMS_PER_THREAD],   ///< [in] The calling thread's input items
-        T (&prev)[ITEMS_PER_THREAD],    ///< [out] The corresponding predecessor items (may be aliased to \p input).  The value \p prev[0] is not updated for <em>thread</em><sub>BLOCK_THREADS-1</sub>.
-        T &block_prefix)                ///< [out] The item \p input[0] from <em>thread</em><sub><tt>0</tt></sub>, provided to all threads
-    {
-        Up(input, prev);
-        block_prefix = temp_storage[BLOCK_THREADS - 1].prev;
-    }
-
-    //@}  end member group
-
-
-};
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/block/block_store.cuh b/pfsimulator/third_party/cub/block/block_store.cuh
deleted file mode 100644
index 648bf9ff4..000000000
--- a/pfsimulator/third_party/cub/block/block_store.cuh
+++ /dev/null
@@ -1,1000 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Operations for writing linear segments of data from the CUDA thread block
- */
-
-#pragma once
-
-#include <iterator>
-
-#include "block_exchange.cuh"
-#include "../util_ptx.cuh"
-#include "../util_macro.cuh"
-#include "../util_type.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \addtogroup UtilIo
- * @{
- */
-
-
-/******************************************************************//**
- * \name Blocked arrangement I/O (direct)
- *********************************************************************/
-//@{
-
-/**
- * \brief Store a blocked arrangement of items across a thread block into a linear segment of items.
- *
- * \blocked
- *
- * \tparam T                    <b>[inferred]</b> The data type to store.
- * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
- * \tparam OutputIteratorT      <b>[inferred]</b> The random-access iterator type for output \iterator.
- */
-template <
-    typename            T,
-    int                 ITEMS_PER_THREAD,
-    typename            OutputIteratorT>
-__device__ __forceinline__ void StoreDirectBlocked(
-    int                 linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
-    OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
-    T                   (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store
-{
-    OutputIteratorT thread_itr = block_itr + (linear_tid * ITEMS_PER_THREAD);
-
-    // Store directly in thread-blocked order
-    #pragma unroll
-    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-    {
-        thread_itr[ITEM] = items[ITEM];
-    }
-}
-
-
-/**
- * \brief Store a blocked arrangement of items across a thread block into a linear segment of items, guarded by range
- *
- * \blocked
- *
- * \tparam T                    <b>[inferred]</b> The data type to store.
- * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
- * \tparam OutputIteratorT      <b>[inferred]</b> The random-access iterator type for output \iterator.
- */
-template <
-    typename            T,
-    int                 ITEMS_PER_THREAD,
-    typename            OutputIteratorT>
-__device__ __forceinline__ void StoreDirectBlocked(
-    int                 linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
-    OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
-    T                   (&items)[ITEMS_PER_THREAD], ///< [in] Data to store
-    int                 valid_items)                ///< [in] Number of valid items to write
-{
-    OutputIteratorT thread_itr = block_itr + (linear_tid * ITEMS_PER_THREAD);
-
-    // Store directly in thread-blocked order
-    #pragma unroll
-    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-    {
-        if (ITEM + (linear_tid * ITEMS_PER_THREAD) < valid_items)
-        {
-            thread_itr[ITEM] = items[ITEM];
-        }
-    }
-}
-
-
-/**
- * \brief Store a blocked arrangement of items across a thread block into a linear segment of items.
- *
- * \blocked
- *
- * The output offset (\p block_ptr + \p block_offset) must be quad-item aligned,
- * which is the default starting offset returned by \p cudaMalloc()
- *
- * \par
- * The following conditions will prevent vectorization and storing will fall back to cub::BLOCK_STORE_DIRECT:
- *   - \p ITEMS_PER_THREAD is odd
- *   - The data type \p T is not a built-in primitive or CUDA vector type (e.g., \p short, \p int2, \p double, \p float2, etc.)
- *
- * \tparam T                    <b>[inferred]</b> The data type to store.
- * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
- *
- */
-template <
-    typename            T,
-    int                 ITEMS_PER_THREAD>
-__device__ __forceinline__ void StoreDirectBlockedVectorized(
-    int                 linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
-    T                   *block_ptr,                 ///< [in] Input pointer for storing from
-    T                   (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store
-{
-    enum
-    {
-        // Maximum CUDA vector size is 4 elements
-        MAX_VEC_SIZE = CUB_MIN(4, ITEMS_PER_THREAD),
-
-        // Vector size must be a power of two and an even divisor of the items per thread
-        VEC_SIZE = ((((MAX_VEC_SIZE - 1) & MAX_VEC_SIZE) == 0) && ((ITEMS_PER_THREAD % MAX_VEC_SIZE) == 0)) ?
-            MAX_VEC_SIZE :
-            1,
-
-        VECTORS_PER_THREAD = ITEMS_PER_THREAD / VEC_SIZE,
-    };
-
-    // Vector type
-    typedef typename CubVector<T, VEC_SIZE>::Type Vector;
-
-    // Alias global pointer
-    Vector *block_ptr_vectors = reinterpret_cast<Vector*>(const_cast<T*>(block_ptr));
-
-    // Alias pointers (use "raw" array here which should get optimized away to prevent conservative PTXAS lmem spilling)
-    Vector raw_vector[VECTORS_PER_THREAD];
-    T *raw_items = reinterpret_cast<T*>(raw_vector);
-
-    // Copy
-    #pragma unroll
-    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-    {
-        raw_items[ITEM] = items[ITEM];
-    }
-
-    // Direct-store using vector types
-    StoreDirectBlocked(linear_tid, block_ptr_vectors, raw_vector);
-}
-
-
-
-//@}  end member group
-/******************************************************************//**
- * \name Striped arrangement I/O (direct)
- *********************************************************************/
-//@{
-
-
-/**
- * \brief Store a striped arrangement of data across the thread block into a linear segment of items.
- *
- * \striped
- *
- * \tparam BLOCK_THREADS        The thread block size in threads
- * \tparam T                    <b>[inferred]</b> The data type to store.
- * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
- * \tparam OutputIteratorT      <b>[inferred]</b> The random-access iterator type for output \iterator.
- */
-template <
-    int                 BLOCK_THREADS,
-    typename            T,
-    int                 ITEMS_PER_THREAD,
-    typename            OutputIteratorT>
-__device__ __forceinline__ void StoreDirectStriped(
-    int                 linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
-    OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
-    T                   (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store
-{
-    OutputIteratorT thread_itr = block_itr + linear_tid;
-
-    // Store directly in striped order
-    #pragma unroll
-    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-    {
-        thread_itr[(ITEM * BLOCK_THREADS)] = items[ITEM];
-    }
-}
-
-
-/**
- * \brief Store a striped arrangement of data across the thread block into a linear segment of items, guarded by range
- *
- * \striped
- *
- * \tparam BLOCK_THREADS        The thread block size in threads
- * \tparam T                    <b>[inferred]</b> The data type to store.
- * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
- * \tparam OutputIteratorT      <b>[inferred]</b> The random-access iterator type for output \iterator.
- */
-template <
-    int                 BLOCK_THREADS,
-    typename            T,
-    int                 ITEMS_PER_THREAD,
-    typename            OutputIteratorT>
-__device__ __forceinline__ void StoreDirectStriped(
-    int                 linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
-    OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
-    T                   (&items)[ITEMS_PER_THREAD], ///< [in] Data to store
-    int                 valid_items)                ///< [in] Number of valid items to write
-{
-    OutputIteratorT thread_itr = block_itr + linear_tid;
-
-    // Store directly in striped order
-    #pragma unroll
-    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-    {
-        if ((ITEM * BLOCK_THREADS) + linear_tid < valid_items)
-        {
-            thread_itr[(ITEM * BLOCK_THREADS)] = items[ITEM];
-        }
-    }
-}
-
-
-
-//@}  end member group
-/******************************************************************//**
- * \name Warp-striped arrangement I/O (direct)
- *********************************************************************/
-//@{
-
-
-/**
- * \brief Store a warp-striped arrangement of data across the thread block into a linear segment of items.
- *
- * \warpstriped
- *
- * \par Usage Considerations
- * The number of threads in the thread block must be a multiple of the architecture's warp size.
- *
- * \tparam T                    <b>[inferred]</b> The data type to store.
- * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
- * \tparam OutputIteratorT      <b>[inferred]</b> The random-access iterator type for output \iterator.
- */
-template <
-    typename            T,
-    int                 ITEMS_PER_THREAD,
-    typename            OutputIteratorT>
-__device__ __forceinline__ void StoreDirectWarpStriped(
-    int                 linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
-    OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
-    T                   (&items)[ITEMS_PER_THREAD]) ///< [out] Data to load
-{
-    int tid         = linear_tid & (CUB_PTX_WARP_THREADS - 1);
-    int wid         = linear_tid >> CUB_PTX_LOG_WARP_THREADS;
-    int warp_offset = wid * CUB_PTX_WARP_THREADS * ITEMS_PER_THREAD;
-
-    OutputIteratorT thread_itr = block_itr + warp_offset + tid;
-
-    // Store directly in warp-striped order
-    #pragma unroll
-    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-    {
-        thread_itr[(ITEM * CUB_PTX_WARP_THREADS)] = items[ITEM];
-    }
-}
-
-
-/**
- * \brief Store a warp-striped arrangement of data across the thread block into a linear segment of items, guarded by range
- *
- * \warpstriped
- *
- * \par Usage Considerations
- * The number of threads in the thread block must be a multiple of the architecture's warp size.
- *
- * \tparam T                    <b>[inferred]</b> The data type to store.
- * \tparam ITEMS_PER_THREAD     <b>[inferred]</b> The number of consecutive items partitioned onto each thread.
- * \tparam OutputIteratorT      <b>[inferred]</b> The random-access iterator type for output \iterator.
- */
-template <
-    typename            T,
-    int                 ITEMS_PER_THREAD,
-    typename            OutputIteratorT>
-__device__ __forceinline__ void StoreDirectWarpStriped(
-    int                 linear_tid,                 ///< [in] A suitable 1D thread-identifier for the calling thread (e.g., <tt>(threadIdx.y * blockDim.x) + linear_tid</tt> for 2D thread blocks)
-    OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
-    T                   (&items)[ITEMS_PER_THREAD], ///< [in] Data to store
-    int                 valid_items)                ///< [in] Number of valid items to write
-{
-    int tid         = linear_tid & (CUB_PTX_WARP_THREADS - 1);
-    int wid         = linear_tid >> CUB_PTX_LOG_WARP_THREADS;
-    int warp_offset = wid * CUB_PTX_WARP_THREADS * ITEMS_PER_THREAD;
-
-    OutputIteratorT thread_itr = block_itr + warp_offset + tid;
-
-    // Store directly in warp-striped order
-    #pragma unroll
-    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ITEM++)
-    {
-        if (warp_offset + tid + (ITEM * CUB_PTX_WARP_THREADS) < valid_items)
-        {
-            thread_itr[(ITEM * CUB_PTX_WARP_THREADS)] = items[ITEM];
-        }
-    }
-}
-
-
-//@}  end member group
-
-
-/** @} */       // end group UtilIo
-
-
-//-----------------------------------------------------------------------------
-// Generic BlockStore abstraction
-//-----------------------------------------------------------------------------
-
-/**
- * \brief cub::BlockStoreAlgorithm enumerates alternative algorithms for cub::BlockStore to write a blocked arrangement of items across a CUDA thread block to a linear segment of memory.
- */
-enum BlockStoreAlgorithm
-{
-    /**
-     * \par Overview
-     *
-     * A [<em>blocked arrangement</em>](index.html#sec5sec3) of data is written
-     * directly to memory.
-     *
-     * \par Performance Considerations
-     * - The utilization of memory transactions (coalescing) decreases as the
-     *   access stride between threads increases (i.e., the number items per thread).
-     */
-    BLOCK_STORE_DIRECT,
-
-    /**
-     * \par Overview
-     *
-     * A [<em>blocked arrangement</em>](index.html#sec5sec3) of data is written directly
-     * to memory using CUDA's built-in vectorized stores as a coalescing optimization.
-     * For example, <tt>st.global.v4.s32</tt> instructions will be generated
-     * when \p T = \p int and \p ITEMS_PER_THREAD % 4 == 0.
-     *
-     * \par Performance Considerations
-     * - The utilization of memory transactions (coalescing) remains high until the the
-     *   access stride between threads (i.e., the number items per thread) exceeds the
-     *   maximum vector store width (typically 4 items or 64B, whichever is lower).
-     * - The following conditions will prevent vectorization and writing will fall back to cub::BLOCK_STORE_DIRECT:
-     *   - \p ITEMS_PER_THREAD is odd
-     *   - The \p OutputIteratorT is not a simple pointer type
-     *   - The block output offset is not quadword-aligned
-     *   - The data type \p T is not a built-in primitive or CUDA vector type (e.g., \p short, \p int2, \p double, \p float2, etc.)
-     */
-    BLOCK_STORE_VECTORIZE,
-
-    /**
-     * \par Overview
-     * A [<em>blocked arrangement</em>](index.html#sec5sec3) is locally
-     * transposed and then efficiently written to memory as a [<em>striped arrangement</em>](index.html#sec5sec3).
-     *
-     * \par Performance Considerations
-     * - The utilization of memory transactions (coalescing) remains high regardless
-     *   of items written per thread.
-     * - The local reordering incurs slightly longer latencies and throughput than the
-     *   direct cub::BLOCK_STORE_DIRECT and cub::BLOCK_STORE_VECTORIZE alternatives.
-     */
-    BLOCK_STORE_TRANSPOSE,
-
-    /**
-     * \par Overview
-     * A [<em>blocked arrangement</em>](index.html#sec5sec3) is locally
-     * transposed and then efficiently written to memory as a
-     * [<em>warp-striped arrangement</em>](index.html#sec5sec3)
-     *
-     * \par Usage Considerations
-     * - BLOCK_THREADS must be a multiple of WARP_THREADS
-     *
-     * \par Performance Considerations
-     * - The utilization of memory transactions (coalescing) remains high regardless
-     *   of items written per thread.
-     * - The local reordering incurs slightly longer latencies and throughput than the
-     *   direct cub::BLOCK_STORE_DIRECT and cub::BLOCK_STORE_VECTORIZE alternatives.
-     */
-    BLOCK_STORE_WARP_TRANSPOSE,
-
-    /**
-     * \par Overview
-     * A [<em>blocked arrangement</em>](index.html#sec5sec3) is locally
-     * transposed and then efficiently written to memory as a
-     * [<em>warp-striped arrangement</em>](index.html#sec5sec3)
-     * To reduce the shared memory requirement, only one warp's worth of shared
-     * memory is provisioned and is subsequently time-sliced among warps.
-     *
-     * \par Usage Considerations
-     * - BLOCK_THREADS must be a multiple of WARP_THREADS
-     *
-     * \par Performance Considerations
-     * - The utilization of memory transactions (coalescing) remains high regardless
-     *   of items written per thread.
-     * - Provisions less shared memory temporary storage, but incurs larger
-     *   latencies than the BLOCK_STORE_WARP_TRANSPOSE alternative.
-     */
-    BLOCK_STORE_WARP_TRANSPOSE_TIMESLICED,
-
-};
-
-
-/**
- * \brief The BlockStore class provides [<em>collective</em>](index.html#sec0) data movement methods for writing a [<em>blocked arrangement</em>](index.html#sec5sec3) of items partitioned across a CUDA thread block to a linear segment of memory.  ![](block_store_logo.png)
- * \ingroup BlockModule
- * \ingroup UtilIo
- *
- * \tparam T                    The type of data to be written.
- * \tparam BLOCK_DIM_X          The thread block length in threads along the X dimension
- * \tparam ITEMS_PER_THREAD     The number of consecutive items partitioned onto each thread.
- * \tparam ALGORITHM            <b>[optional]</b> cub::BlockStoreAlgorithm tuning policy enumeration.  default: cub::BLOCK_STORE_DIRECT.
- * \tparam WARP_TIME_SLICING    <b>[optional]</b> Whether or not only one warp's worth of shared memory should be allocated and time-sliced among block-warps during any load-related data transpositions (versus each warp having its own storage). (default: false)
- * \tparam BLOCK_DIM_Y          <b>[optional]</b> The thread block length in threads along the Y dimension (default: 1)
- * \tparam BLOCK_DIM_Z          <b>[optional]</b> The thread block length in threads along the Z dimension (default: 1)
- * \tparam PTX_ARCH             <b>[optional]</b> \ptxversion
- *
- * \par Overview
- * - The BlockStore class provides a single data movement abstraction that can be specialized
- *   to implement different cub::BlockStoreAlgorithm strategies.  This facilitates different
- *   performance policies for different architectures, data types, granularity sizes, etc.
- * - BlockStore can be optionally specialized by different data movement strategies:
- *   -# <b>cub::BLOCK_STORE_DIRECT</b>.  A [<em>blocked arrangement</em>](index.html#sec5sec3) of data is written
- *      directly to memory. [More...](\ref cub::BlockStoreAlgorithm)
- *   -# <b>cub::BLOCK_STORE_VECTORIZE</b>.  A [<em>blocked arrangement</em>](index.html#sec5sec3)
- *      of data is written directly to memory using CUDA's built-in vectorized stores as a
- *      coalescing optimization.  [More...](\ref cub::BlockStoreAlgorithm)
- *   -# <b>cub::BLOCK_STORE_TRANSPOSE</b>.  A [<em>blocked arrangement</em>](index.html#sec5sec3)
- *      is locally transposed into a [<em>striped arrangement</em>](index.html#sec5sec3) which is
- *      then written to memory.  [More...](\ref cub::BlockStoreAlgorithm)
- *   -# <b>cub::BLOCK_STORE_WARP_TRANSPOSE</b>.  A [<em>blocked arrangement</em>](index.html#sec5sec3)
- *      is locally transposed into a [<em>warp-striped arrangement</em>](index.html#sec5sec3) which is
- *      then written to memory.  [More...](\ref cub::BlockStoreAlgorithm)
- * - \rowmajor
- *
- * \par A Simple Example
- * \blockcollective{BlockStore}
- * \par
- * The code snippet below illustrates the storing of a "blocked" arrangement
- * of 512 integers across 128 threads (where each thread owns 4 consecutive items)
- * into a linear segment of memory.  The store is specialized for \p BLOCK_STORE_WARP_TRANSPOSE,
- * meaning items are locally reordered among threads so that memory references will be
- * efficiently coalesced using a warp-striped access pattern.
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/block/block_store.cuh>
- *
- * __global__ void ExampleKernel(int *d_data, ...)
- * {
- *     // Specialize BlockStore for a 1D block of 128 threads owning 4 integer items each
- *     typedef cub::BlockStore<int, 128, 4, BLOCK_STORE_WARP_TRANSPOSE> BlockStore;
- *
- *     // Allocate shared memory for BlockStore
- *     __shared__ typename BlockStore::TempStorage temp_storage;
- *
- *     // Obtain a segment of consecutive items that are blocked across threads
- *     int thread_data[4];
- *     ...
- *
- *     // Store items to linear memory
- *     int thread_data[4];
- *     BlockStore(temp_storage).Store(d_data, thread_data);
- *
- * \endcode
- * \par
- * Suppose the set of \p thread_data across the block of threads is
- * <tt>{ [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }</tt>.
- * The output \p d_data will be <tt>0, 1, 2, 3, 4, 5, ...</tt>.
- *
- */
-template <
-    typename                T,
-    int                     BLOCK_DIM_X,
-    int                     ITEMS_PER_THREAD,
-    BlockStoreAlgorithm     ALGORITHM           = BLOCK_STORE_DIRECT,
-    int                     BLOCK_DIM_Y         = 1,
-    int                     BLOCK_DIM_Z         = 1,
-    int                     PTX_ARCH            = CUB_PTX_ARCH>
-class BlockStore
-{
-private:
-    /******************************************************************************
-     * Constants and typed definitions
-     ******************************************************************************/
-
-    /// Constants
-    enum
-    {
-        /// The thread block size in threads
-        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-    };
-
-
-    /******************************************************************************
-     * Algorithmic variants
-     ******************************************************************************/
-
-    /// Store helper
-    template <BlockStoreAlgorithm _POLICY, int DUMMY>
-    struct StoreInternal;
-
-
-    /**
-     * BLOCK_STORE_DIRECT specialization of store helper
-     */
-    template <int DUMMY>
-    struct StoreInternal<BLOCK_STORE_DIRECT, DUMMY>
-    {
-        /// Shared memory storage layout type
-        typedef NullType TempStorage;
-
-        /// Linear thread-id
-        int linear_tid;
-
-        /// Constructor
-        __device__ __forceinline__ StoreInternal(
-            TempStorage &/*temp_storage*/,
-            int linear_tid)
-        :
-            linear_tid(linear_tid)
-        {}
-
-        /// Store items into a linear segment of memory
-        template <typename OutputIteratorT>
-        __device__ __forceinline__ void Store(
-            OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
-            T                   (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store
-        {
-            StoreDirectBlocked(linear_tid, block_itr, items);
-        }
-
-        /// Store items into a linear segment of memory, guarded by range
-        template <typename OutputIteratorT>
-        __device__ __forceinline__ void Store(
-            OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
-            T                   (&items)[ITEMS_PER_THREAD], ///< [in] Data to store
-            int                 valid_items)                ///< [in] Number of valid items to write
-        {
-            StoreDirectBlocked(linear_tid, block_itr, items, valid_items);
-        }
-    };
-
-
-    /**
-     * BLOCK_STORE_VECTORIZE specialization of store helper
-     */
-    template <int DUMMY>
-    struct StoreInternal<BLOCK_STORE_VECTORIZE, DUMMY>
-    {
-        /// Shared memory storage layout type
-        typedef NullType TempStorage;
-
-        /// Linear thread-id
-        int linear_tid;
-
-        /// Constructor
-        __device__ __forceinline__ StoreInternal(
-            TempStorage &/*temp_storage*/,
-            int linear_tid)
-        :
-            linear_tid(linear_tid)
-        {}
-
-        /// Store items into a linear segment of memory, specialized for native pointer types (attempts vectorization)
-        __device__ __forceinline__ void Store(
-            T                   *block_ptr,                 ///< [in] The thread block's base output iterator for storing to
-            T                   (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store
-        {
-            StoreDirectBlockedVectorized(linear_tid, block_ptr, items);
-        }
-
-        /// Store items into a linear segment of memory, specialized for opaque input iterators (skips vectorization)
-        template <typename OutputIteratorT>
-        __device__ __forceinline__ void Store(
-            OutputIteratorT    block_itr,                  ///< [in] The thread block's base output iterator for storing to
-            T                   (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store
-        {
-            StoreDirectBlocked(linear_tid, block_itr, items);
-        }
-
-        /// Store items into a linear segment of memory, guarded by range
-        template <typename OutputIteratorT>
-        __device__ __forceinline__ void Store(
-            OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
-            T                   (&items)[ITEMS_PER_THREAD], ///< [in] Data to store
-            int                 valid_items)                ///< [in] Number of valid items to write
-        {
-            StoreDirectBlocked(linear_tid, block_itr, items, valid_items);
-        }
-    };
-
-
-    /**
-     * BLOCK_STORE_TRANSPOSE specialization of store helper
-     */
-    template <int DUMMY>
-    struct StoreInternal<BLOCK_STORE_TRANSPOSE, DUMMY>
-    {
-        // BlockExchange utility type for keys
-        typedef BlockExchange<T, BLOCK_DIM_X, ITEMS_PER_THREAD, false, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> BlockExchange;
-
-        /// Shared memory storage layout type
-        struct _TempStorage : BlockExchange::TempStorage
-        {
-            /// Temporary storage for partially-full block guard
-            volatile int valid_items;
-        };
-
-        /// Alias wrapper allowing storage to be unioned
-        struct TempStorage : Uninitialized<_TempStorage> {};
-
-        /// Thread reference to shared storage
-        _TempStorage &temp_storage;
-
-        /// Linear thread-id
-        int linear_tid;
-
-        /// Constructor
-        __device__ __forceinline__ StoreInternal(
-            TempStorage &temp_storage,
-            int linear_tid)
-        :
-            temp_storage(temp_storage.Alias()),
-            linear_tid(linear_tid)
-        {}
-
-        /// Store items into a linear segment of memory
-        template <typename OutputIteratorT>
-        __device__ __forceinline__ void Store(
-            OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
-            T                   (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store
-        {
-            BlockExchange(temp_storage).BlockedToStriped(items);
-            StoreDirectStriped<BLOCK_THREADS>(linear_tid, block_itr, items);
-        }
-
-        /// Store items into a linear segment of memory, guarded by range
-        template <typename OutputIteratorT>
-        __device__ __forceinline__ void Store(
-            OutputIteratorT   block_itr,                  ///< [in] The thread block's base output iterator for storing to
-            T                   (&items)[ITEMS_PER_THREAD], ///< [in] Data to store
-            int                 valid_items)                ///< [in] Number of valid items to write
-        {
-            BlockExchange(temp_storage).BlockedToStriped(items);
-            if (linear_tid == 0)
-                temp_storage.valid_items = valid_items;     // Move through volatile smem as a workaround to prevent RF spilling on subsequent loads
-            CTA_SYNC();
-            StoreDirectStriped<BLOCK_THREADS>(linear_tid, block_itr, items, temp_storage.valid_items);
-        }
-    };
-
-
-    /**
-     * BLOCK_STORE_WARP_TRANSPOSE specialization of store helper
-     */
-    template <int DUMMY>
-    struct StoreInternal<BLOCK_STORE_WARP_TRANSPOSE, DUMMY>
-    {
-        enum
-        {
-            WARP_THREADS = CUB_WARP_THREADS(PTX_ARCH)
-        };
-
-        // Assert BLOCK_THREADS must be a multiple of WARP_THREADS
-        CUB_STATIC_ASSERT((BLOCK_THREADS % WARP_THREADS == 0), "BLOCK_THREADS must be a multiple of WARP_THREADS");
-
-        // BlockExchange utility type for keys
-        typedef BlockExchange<T, BLOCK_DIM_X, ITEMS_PER_THREAD, false, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> BlockExchange;
-
-        /// Shared memory storage layout type
-        struct _TempStorage : BlockExchange::TempStorage
-        {
-            /// Temporary storage for partially-full block guard
-            volatile int valid_items;
-        };
-
-        /// Alias wrapper allowing storage to be unioned
-        struct TempStorage : Uninitialized<_TempStorage> {};
-
-        /// Thread reference to shared storage
-        _TempStorage &temp_storage;
-
-        /// Linear thread-id
-        int linear_tid;
-
-        /// Constructor
-        __device__ __forceinline__ StoreInternal(
-            TempStorage &temp_storage,
-            int linear_tid)
-        :
-            temp_storage(temp_storage.Alias()),
-            linear_tid(linear_tid)
-        {}
-
-        /// Store items into a linear segment of memory
-        template <typename OutputIteratorT>
-        __device__ __forceinline__ void Store(
-            OutputIteratorT   block_itr,                    ///< [in] The thread block's base output iterator for storing to
-            T                 (&items)[ITEMS_PER_THREAD])   ///< [in] Data to store
-        {
-            BlockExchange(temp_storage).BlockedToWarpStriped(items);
-            StoreDirectWarpStriped(linear_tid, block_itr, items);
-        }
-
-        /// Store items into a linear segment of memory, guarded by range
-        template <typename OutputIteratorT>
-        __device__ __forceinline__ void Store(
-            OutputIteratorT   block_itr,                    ///< [in] The thread block's base output iterator for storing to
-            T                 (&items)[ITEMS_PER_THREAD],   ///< [in] Data to store
-            int               valid_items)                  ///< [in] Number of valid items to write
-        {
-            BlockExchange(temp_storage).BlockedToWarpStriped(items);
-            if (linear_tid == 0)
-                temp_storage.valid_items = valid_items;     // Move through volatile smem as a workaround to prevent RF spilling on subsequent loads
-            CTA_SYNC();
-            StoreDirectWarpStriped(linear_tid, block_itr, items, temp_storage.valid_items);
-        }
-    };
-
-
-    /**
-     * BLOCK_STORE_WARP_TRANSPOSE_TIMESLICED specialization of store helper
-     */
-    template <int DUMMY>
-    struct StoreInternal<BLOCK_STORE_WARP_TRANSPOSE_TIMESLICED, DUMMY>
-    {
-        enum
-        {
-            WARP_THREADS = CUB_WARP_THREADS(PTX_ARCH)
-        };
-
-        // Assert BLOCK_THREADS must be a multiple of WARP_THREADS
-        CUB_STATIC_ASSERT((BLOCK_THREADS % WARP_THREADS == 0), "BLOCK_THREADS must be a multiple of WARP_THREADS");
-
-        // BlockExchange utility type for keys
-        typedef BlockExchange<T, BLOCK_DIM_X, ITEMS_PER_THREAD, true, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> BlockExchange;
-
-        /// Shared memory storage layout type
-        struct _TempStorage : BlockExchange::TempStorage
-        {
-            /// Temporary storage for partially-full block guard
-            volatile int valid_items;
-        };
-
-        /// Alias wrapper allowing storage to be unioned
-        struct TempStorage : Uninitialized<_TempStorage> {};
-
-        /// Thread reference to shared storage
-        _TempStorage &temp_storage;
-
-        /// Linear thread-id
-        int linear_tid;
-
-        /// Constructor
-        __device__ __forceinline__ StoreInternal(
-            TempStorage &temp_storage,
-            int linear_tid)
-        :
-            temp_storage(temp_storage.Alias()),
-            linear_tid(linear_tid)
-        {}
-
-        /// Store items into a linear segment of memory
-        template <typename OutputIteratorT>
-        __device__ __forceinline__ void Store(
-            OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
-            T                   (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store
-        {
-            BlockExchange(temp_storage).BlockedToWarpStriped(items);
-            StoreDirectWarpStriped(linear_tid, block_itr, items);
-        }
-
-        /// Store items into a linear segment of memory, guarded by range
-        template <typename OutputIteratorT>
-        __device__ __forceinline__ void Store(
-            OutputIteratorT   block_itr,                  ///< [in] The thread block's base output iterator for storing to
-            T                   (&items)[ITEMS_PER_THREAD], ///< [in] Data to store
-            int                 valid_items)                ///< [in] Number of valid items to write
-        {
-            BlockExchange(temp_storage).BlockedToWarpStriped(items);
-            if (linear_tid == 0)
-                temp_storage.valid_items = valid_items;     // Move through volatile smem as a workaround to prevent RF spilling on subsequent loads
-            CTA_SYNC();
-            StoreDirectWarpStriped(linear_tid, block_itr, items, temp_storage.valid_items);
-        }
-    };
-
-    /******************************************************************************
-     * Type definitions
-     ******************************************************************************/
-
-    /// Internal load implementation to use
-    typedef StoreInternal<ALGORITHM, 0> InternalStore;
-
-
-    /// Shared memory storage layout type
-    typedef typename InternalStore::TempStorage _TempStorage;
-
-
-    /******************************************************************************
-     * Utility methods
-     ******************************************************************************/
-
-    /// Internal storage allocator
-    __device__ __forceinline__ _TempStorage& PrivateStorage()
-    {
-        __shared__ _TempStorage private_storage;
-        return private_storage;
-    }
-
-
-    /******************************************************************************
-     * Thread fields
-     ******************************************************************************/
-
-    /// Thread reference to shared storage
-    _TempStorage &temp_storage;
-
-    /// Linear thread-id
-    int linear_tid;
-
-public:
-
-
-    /// \smemstorage{BlockStore}
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    /******************************************************************//**
-     * \name Collective constructors
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Collective constructor using a private static allocation of shared memory as temporary storage.
-     */
-    __device__ __forceinline__ BlockStore()
-    :
-        temp_storage(PrivateStorage()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    /**
-     * \brief Collective constructor using the specified memory allocation as temporary storage.
-     */
-    __device__ __forceinline__ BlockStore(
-        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Data movement
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Store items into a linear segment of memory.
-     *
-     * \par
-     * - \blocked
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates the storing of a "blocked" arrangement
-     * of 512 integers across 128 threads (where each thread owns 4 consecutive items)
-     * into a linear segment of memory.  The store is specialized for \p BLOCK_STORE_WARP_TRANSPOSE,
-     * meaning items are locally reordered among threads so that memory references will be
-     * efficiently coalesced using a warp-striped access pattern.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_store.cuh>
-     *
-     * __global__ void ExampleKernel(int *d_data, ...)
-     * {
-     *     // Specialize BlockStore for a 1D block of 128 threads owning 4 integer items each
-     *     typedef cub::BlockStore<int, 128, 4, BLOCK_STORE_WARP_TRANSPOSE> BlockStore;
-     *
-     *     // Allocate shared memory for BlockStore
-     *     __shared__ typename BlockStore::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Store items to linear memory
-     *     int thread_data[4];
-     *     BlockStore(temp_storage).Store(d_data, thread_data);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of \p thread_data across the block of threads is
-     * <tt>{ [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }</tt>.
-     * The output \p d_data will be <tt>0, 1, 2, 3, 4, 5, ...</tt>.
-     *
-     */
-    template <typename OutputIteratorT>
-    __device__ __forceinline__ void Store(
-        OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
-        T                   (&items)[ITEMS_PER_THREAD]) ///< [in] Data to store
-    {
-        InternalStore(temp_storage, linear_tid).Store(block_itr, items);
-    }
-
-    /**
-     * \brief Store items into a linear segment of memory, guarded by range.
-     *
-     * \par
-     * - \blocked
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates the guarded storing of a "blocked" arrangement
-     * of 512 integers across 128 threads (where each thread owns 4 consecutive items)
-     * into a linear segment of memory.  The store is specialized for \p BLOCK_STORE_WARP_TRANSPOSE,
-     * meaning items are locally reordered among threads so that memory references will be
-     * efficiently coalesced using a warp-striped access pattern.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/block/block_store.cuh>
-     *
-     * __global__ void ExampleKernel(int *d_data, int valid_items, ...)
-     * {
-     *     // Specialize BlockStore for a 1D block of 128 threads owning 4 integer items each
-     *     typedef cub::BlockStore<int, 128, 4, BLOCK_STORE_WARP_TRANSPOSE> BlockStore;
-     *
-     *     // Allocate shared memory for BlockStore
-     *     __shared__ typename BlockStore::TempStorage temp_storage;
-     *
-     *     // Obtain a segment of consecutive items that are blocked across threads
-     *     int thread_data[4];
-     *     ...
-     *
-     *     // Store items to linear memory
-     *     int thread_data[4];
-     *     BlockStore(temp_storage).Store(d_data, thread_data, valid_items);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of \p thread_data across the block of threads is
-     * <tt>{ [0,1,2,3], [4,5,6,7], ..., [508,509,510,511] }</tt> and \p valid_items is \p 5.
-     * The output \p d_data will be <tt>0, 1, 2, 3, 4, ?, ?, ?, ...</tt>, with
-     * only the first two threads being unmasked to store portions of valid data.
-     *
-     */
-    template <typename OutputIteratorT>
-    __device__ __forceinline__ void Store(
-        OutputIteratorT     block_itr,                  ///< [in] The thread block's base output iterator for storing to
-        T                   (&items)[ITEMS_PER_THREAD], ///< [in] Data to store
-        int                 valid_items)                ///< [in] Number of valid items to write
-    {
-        InternalStore(temp_storage, linear_tid).Store(block_itr, items, valid_items);
-    }
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/block/specializations/block_histogram_atomic.cuh b/pfsimulator/third_party/cub/block/specializations/block_histogram_atomic.cuh
deleted file mode 100644
index 29db0df71..000000000
--- a/pfsimulator/third_party/cub/block/specializations/block_histogram_atomic.cuh
+++ /dev/null
@@ -1,82 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * The cub::BlockHistogramAtomic class provides atomic-based methods for constructing block-wide histograms from data samples partitioned across a CUDA thread block.
- */
-
-#pragma once
-
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \brief The BlockHistogramAtomic class provides atomic-based methods for constructing block-wide histograms from data samples partitioned across a CUDA thread block.
- */
-template <int BINS>
-struct BlockHistogramAtomic
-{
-    /// Shared memory storage layout type
-    struct TempStorage {};
-
-
-    /// Constructor
-    __device__ __forceinline__ BlockHistogramAtomic(
-        TempStorage &temp_storage)
-    {}
-
-
-    /// Composite data onto an existing histogram
-    template <
-        typename            T,
-        typename            CounterT,     
-        int                 ITEMS_PER_THREAD>
-    __device__ __forceinline__ void Composite(
-        T                   (&items)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input values to histogram
-        CounterT             histogram[BINS])                 ///< [out] Reference to shared/device-accessible memory histogram
-    {
-        // Update histogram
-        #pragma unroll
-        for (int i = 0; i < ITEMS_PER_THREAD; ++i)
-        {
-              atomicAdd(histogram + items[i], 1);
-        }
-    }
-
-};
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/block/specializations/block_histogram_sort.cuh b/pfsimulator/third_party/cub/block/specializations/block_histogram_sort.cuh
deleted file mode 100644
index 9ef417adc..000000000
--- a/pfsimulator/third_party/cub/block/specializations/block_histogram_sort.cuh
+++ /dev/null
@@ -1,226 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * The cub::BlockHistogramSort class provides sorting-based methods for constructing block-wide histograms from data samples partitioned across a CUDA thread block.
- */
-
-#pragma once
-
-#include "../../block/block_radix_sort.cuh"
-#include "../../block/block_discontinuity.cuh"
-#include "../../util_ptx.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-
-/**
- * \brief The BlockHistogramSort class provides sorting-based methods for constructing block-wide histograms from data samples partitioned across a CUDA thread block.
- */
-template <
-    typename    T,                  ///< Sample type
-    int         BLOCK_DIM_X,        ///< The thread block length in threads along the X dimension
-    int         ITEMS_PER_THREAD,   ///< The number of samples per thread
-    int         BINS,               ///< The number of bins into which histogram samples may fall
-    int         BLOCK_DIM_Y,        ///< The thread block length in threads along the Y dimension
-    int         BLOCK_DIM_Z,        ///< The thread block length in threads along the Z dimension
-    int         PTX_ARCH>           ///< The PTX compute capability for which to to specialize this collective
-struct BlockHistogramSort
-{
-    /// Constants
-    enum
-    {
-        /// The thread block size in threads
-        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-    };
-
-    // Parameterize BlockRadixSort type for our thread block
-    typedef BlockRadixSort<
-            T,
-            BLOCK_DIM_X,
-            ITEMS_PER_THREAD,
-            NullType,
-            4,
-            (PTX_ARCH >= 350) ? true : false,
-            BLOCK_SCAN_WARP_SCANS,
-            cudaSharedMemBankSizeFourByte,
-            BLOCK_DIM_Y,
-            BLOCK_DIM_Z,
-            PTX_ARCH>
-        BlockRadixSortT;
-
-    // Parameterize BlockDiscontinuity type for our thread block
-    typedef BlockDiscontinuity<
-            T,
-            BLOCK_DIM_X,
-            BLOCK_DIM_Y,
-            BLOCK_DIM_Z,
-            PTX_ARCH>
-        BlockDiscontinuityT;
-
-    /// Shared memory
-    union _TempStorage
-    {
-        // Storage for sorting bin values
-        typename BlockRadixSortT::TempStorage sort;
-
-        struct
-        {
-            // Storage for detecting discontinuities in the tile of sorted bin values
-            typename BlockDiscontinuityT::TempStorage flag;
-
-            // Storage for noting begin/end offsets of bin runs in the tile of sorted bin values
-            unsigned int run_begin[BINS];
-            unsigned int run_end[BINS];
-        };
-    };
-
-
-    /// Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    // Thread fields
-    _TempStorage &temp_storage;
-    unsigned int linear_tid;
-
-
-    /// Constructor
-    __device__ __forceinline__ BlockHistogramSort(
-        TempStorage     &temp_storage)
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    // Discontinuity functor
-    struct DiscontinuityOp
-    {
-        // Reference to temp_storage
-        _TempStorage &temp_storage;
-
-        // Constructor
-        __device__ __forceinline__ DiscontinuityOp(_TempStorage &temp_storage) :
-            temp_storage(temp_storage)
-        {}
-
-        // Discontinuity predicate
-        __device__ __forceinline__ bool operator()(const T &a, const T &b, int b_index)
-        {
-            if (a != b)
-            {
-                // Note the begin/end offsets in shared storage
-                temp_storage.run_begin[b] = b_index;
-                temp_storage.run_end[a] = b_index;
-
-                return true;
-            }
-            else
-            {
-                return false;
-            }
-        }
-    };
-
-
-    // Composite data onto an existing histogram
-    template <
-        typename            CounterT     >
-    __device__ __forceinline__ void Composite(
-        T                   (&items)[ITEMS_PER_THREAD],     ///< [in] Calling thread's input values to histogram
-        CounterT            histogram[BINS])                 ///< [out] Reference to shared/device-accessible memory histogram
-    {
-        enum { TILE_SIZE = BLOCK_THREADS * ITEMS_PER_THREAD };
-
-        // Sort bytes in blocked arrangement
-        BlockRadixSortT(temp_storage.sort).Sort(items);
-
-        CTA_SYNC();
-
-        // Initialize the shared memory's run_begin and run_end for each bin
-        int histo_offset = 0;
-
-        #pragma unroll
-        for(; histo_offset + BLOCK_THREADS <= BINS; histo_offset += BLOCK_THREADS)
-        {
-            temp_storage.run_begin[histo_offset + linear_tid] = TILE_SIZE;
-            temp_storage.run_end[histo_offset + linear_tid] = TILE_SIZE;
-        }
-        // Finish up with guarded initialization if necessary
-        if ((BINS % BLOCK_THREADS != 0) && (histo_offset + linear_tid < BINS))
-        {
-            temp_storage.run_begin[histo_offset + linear_tid] = TILE_SIZE;
-            temp_storage.run_end[histo_offset + linear_tid] = TILE_SIZE;
-        }
-
-        CTA_SYNC();
-
-        int flags[ITEMS_PER_THREAD];    // unused
-
-        // Compute head flags to demarcate contiguous runs of the same bin in the sorted tile
-        DiscontinuityOp flag_op(temp_storage);
-        BlockDiscontinuityT(temp_storage.flag).FlagHeads(flags, items, flag_op);
-
-        // Update begin for first item
-        if (linear_tid == 0) temp_storage.run_begin[items[0]] = 0;
-
-        CTA_SYNC();
-
-        // Composite into histogram
-        histo_offset = 0;
-
-        #pragma unroll
-        for(; histo_offset + BLOCK_THREADS <= BINS; histo_offset += BLOCK_THREADS)
-        {
-            int thread_offset = histo_offset + linear_tid;
-            CounterT      count = temp_storage.run_end[thread_offset] - temp_storage.run_begin[thread_offset];
-            histogram[thread_offset] += count;
-        }
-
-        // Finish up with guarded composition if necessary
-        if ((BINS % BLOCK_THREADS != 0) && (histo_offset + linear_tid < BINS))
-        {
-            int thread_offset = histo_offset + linear_tid;
-            CounterT      count = temp_storage.run_end[thread_offset] - temp_storage.run_begin[thread_offset];
-            histogram[thread_offset] += count;
-        }
-    }
-
-};
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/block/specializations/block_reduce_raking.cuh b/pfsimulator/third_party/cub/block/specializations/block_reduce_raking.cuh
deleted file mode 100644
index aff97fc9b..000000000
--- a/pfsimulator/third_party/cub/block/specializations/block_reduce_raking.cuh
+++ /dev/null
@@ -1,226 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::BlockReduceRaking provides raking-based methods of parallel reduction across a CUDA thread block.  Supports non-commutative reduction operators.
- */
-
-#pragma once
-
-#include "../../block/block_raking_layout.cuh"
-#include "../../warp/warp_reduce.cuh"
-#include "../../thread/thread_reduce.cuh"
-#include "../../util_ptx.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \brief BlockReduceRaking provides raking-based methods of parallel reduction across a CUDA thread block.  Supports non-commutative reduction operators.
- *
- * Supports non-commutative binary reduction operators.  Unlike commutative
- * reduction operators (e.g., addition), the application of a non-commutative
- * reduction operator (e.g, string concatenation) across a sequence of inputs must
- * honor the relative ordering of items and partial reductions when applying the
- * reduction operator.
- *
- * Compared to the implementation of BlockReduceRaking (which does not support
- * non-commutative operators), this implementation requires a few extra
- * rounds of inter-thread communication.
- */
-template <
-    typename    T,              ///< Data type being reduced
-    int         BLOCK_DIM_X,    ///< The thread block length in threads along the X dimension
-    int         BLOCK_DIM_Y,    ///< The thread block length in threads along the Y dimension
-    int         BLOCK_DIM_Z,    ///< The thread block length in threads along the Z dimension
-    int         PTX_ARCH>       ///< The PTX compute capability for which to to specialize this collective
-struct BlockReduceRaking
-{
-    /// Constants
-    enum
-    {
-        /// The thread block size in threads
-        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-    };
-
-    /// Layout type for padded thread block raking grid
-    typedef BlockRakingLayout<T, BLOCK_THREADS, PTX_ARCH> BlockRakingLayout;
-
-    ///  WarpReduce utility type
-    typedef typename WarpReduce<T, BlockRakingLayout::RAKING_THREADS, PTX_ARCH>::InternalWarpReduce WarpReduce;
-
-    /// Constants
-    enum
-    {
-        /// Number of raking threads
-        RAKING_THREADS = BlockRakingLayout::RAKING_THREADS,
-
-        /// Number of raking elements per warp synchronous raking thread
-        SEGMENT_LENGTH = BlockRakingLayout::SEGMENT_LENGTH,
-
-        /// Cooperative work can be entirely warp synchronous
-        WARP_SYNCHRONOUS = (RAKING_THREADS == BLOCK_THREADS),
-
-        /// Whether or not warp-synchronous reduction should be unguarded (i.e., the warp-reduction elements is a power of two
-        WARP_SYNCHRONOUS_UNGUARDED = PowerOfTwo<RAKING_THREADS>::VALUE,
-
-        /// Whether or not accesses into smem are unguarded
-        RAKING_UNGUARDED = BlockRakingLayout::UNGUARDED,
-
-    };
-
-
-    /// Shared memory storage layout type
-    union _TempStorage
-    {
-        typename WarpReduce::TempStorage            warp_storage;        ///< Storage for warp-synchronous reduction
-        typename BlockRakingLayout::TempStorage     raking_grid;         ///< Padded thread block raking grid
-    };
-
-
-    /// Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    // Thread fields
-    _TempStorage &temp_storage;
-    unsigned int linear_tid;
-
-
-    /// Constructor
-    __device__ __forceinline__ BlockReduceRaking(
-        TempStorage &temp_storage)
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    template <bool IS_FULL_TILE, typename ReductionOp, int ITERATION>
-    __device__ __forceinline__ T RakingReduction(
-        ReductionOp                 reduction_op,       ///< [in] Binary scan operator
-        T                           *raking_segment,
-        T                           partial,            ///< [in] <b>[<em>lane</em><sub>0</sub> only]</b> Warp-wide aggregate reduction of input items
-        int                         num_valid,          ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
-        Int2Type<ITERATION>         /*iteration*/)
-    {
-        // Update partial if addend is in range
-        if ((IS_FULL_TILE && RAKING_UNGUARDED) || ((linear_tid * SEGMENT_LENGTH) + ITERATION < num_valid))
-        {
-            T addend = raking_segment[ITERATION];
-            partial = reduction_op(partial, addend);
-        }
-        return RakingReduction<IS_FULL_TILE>(reduction_op, raking_segment, partial, num_valid, Int2Type<ITERATION + 1>());
-    }
-
-    template <bool IS_FULL_TILE, typename ReductionOp>
-    __device__ __forceinline__ T RakingReduction(
-        ReductionOp                 /*reduction_op*/,   ///< [in] Binary scan operator
-        T                           * /*raking_segment*/,
-        T                           partial,            ///< [in] <b>[<em>lane</em><sub>0</sub> only]</b> Warp-wide aggregate reduction of input items
-        int                         /*num_valid*/,      ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
-        Int2Type<SEGMENT_LENGTH>    /*iteration*/)
-    {
-        return partial;
-    }
-
-
-
-    /// Computes a thread block-wide reduction using the specified reduction operator. The first num_valid threads each contribute one reduction partial.  The return value is only valid for thread<sub>0</sub>.
-    template <
-        bool                IS_FULL_TILE,
-        typename            ReductionOp>
-    __device__ __forceinline__ T Reduce(
-        T                   partial,            ///< [in] Calling thread's input partial reductions
-        int                 num_valid,          ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
-        ReductionOp         reduction_op)       ///< [in] Binary reduction operator
-    {
-        if (WARP_SYNCHRONOUS)
-        {
-            // Short-circuit directly to warp synchronous reduction (unguarded if active threads is a power-of-two)
-            partial = WarpReduce(temp_storage.warp_storage).template Reduce<IS_FULL_TILE>(
-                partial,
-                num_valid,
-                reduction_op);
-        }
-        else
-        {
-            // Place partial into shared memory grid.
-            *BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid) = partial;
-
-            CTA_SYNC();
-
-            // Reduce parallelism to one warp
-            if (linear_tid < RAKING_THREADS)
-            {
-                // Raking reduction in grid
-                T *raking_segment = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid);
-                partial = raking_segment[0];
-
-                partial = RakingReduction<IS_FULL_TILE>(reduction_op, raking_segment, partial, num_valid, Int2Type<1>());
-
-                int valid_raking_threads = (IS_FULL_TILE) ?
-                    RAKING_THREADS :
-                    (num_valid + SEGMENT_LENGTH - 1) / SEGMENT_LENGTH;
-
-                partial = WarpReduce(temp_storage.warp_storage).template Reduce<IS_FULL_TILE && RAKING_UNGUARDED>(
-                    partial,
-                    valid_raking_threads,
-                    reduction_op);
-
-            }
-        }
-
-        return partial;
-    }
-
-
-    /// Computes a thread block-wide reduction using addition (+) as the reduction operator. The first num_valid threads each contribute one reduction partial.  The return value is only valid for thread<sub>0</sub>.
-    template <bool IS_FULL_TILE>
-    __device__ __forceinline__ T Sum(
-        T                   partial,            ///< [in] Calling thread's input partial reductions
-        int                 num_valid)          ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
-    {
-        cub::Sum reduction_op;
-
-        return Reduce<IS_FULL_TILE>(partial, num_valid, reduction_op);
-    }
-
-
-
-};
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/block/specializations/block_reduce_raking_commutative_only.cuh b/pfsimulator/third_party/cub/block/specializations/block_reduce_raking_commutative_only.cuh
deleted file mode 100644
index 454fdafa5..000000000
--- a/pfsimulator/third_party/cub/block/specializations/block_reduce_raking_commutative_only.cuh
+++ /dev/null
@@ -1,199 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::BlockReduceRakingCommutativeOnly provides raking-based methods of parallel reduction across a CUDA thread block.  Does not support non-commutative reduction operators.
- */
-
-#pragma once
-
-#include "block_reduce_raking.cuh"
-#include "../../warp/warp_reduce.cuh"
-#include "../../thread/thread_reduce.cuh"
-#include "../../util_ptx.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \brief BlockReduceRakingCommutativeOnly provides raking-based methods of parallel reduction across a CUDA thread block.  Does not support non-commutative reduction operators.  Does not support block sizes that are not a multiple of the warp size.
- */
-template <
-    typename    T,              ///< Data type being reduced
-    int         BLOCK_DIM_X,    ///< The thread block length in threads along the X dimension
-    int         BLOCK_DIM_Y,    ///< The thread block length in threads along the Y dimension
-    int         BLOCK_DIM_Z,    ///< The thread block length in threads along the Z dimension
-    int         PTX_ARCH>       ///< The PTX compute capability for which to to specialize this collective
-struct BlockReduceRakingCommutativeOnly
-{
-    /// Constants
-    enum
-    {
-        /// The thread block size in threads
-        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-    };
-
-    // The fall-back implementation to use when BLOCK_THREADS is not a multiple of the warp size or not all threads have valid values
-    typedef BlockReduceRaking<T, BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z, PTX_ARCH> FallBack;
-
-    /// Constants
-    enum
-    {
-        /// Number of warp threads
-        WARP_THREADS = CUB_WARP_THREADS(PTX_ARCH),
-
-        /// Whether or not to use fall-back
-        USE_FALLBACK = ((BLOCK_THREADS % WARP_THREADS != 0) || (BLOCK_THREADS <= WARP_THREADS)),
-
-        /// Number of raking threads
-        RAKING_THREADS = WARP_THREADS,
-
-        /// Number of threads actually sharing items with the raking threads
-        SHARING_THREADS = CUB_MAX(1, BLOCK_THREADS - RAKING_THREADS),
-
-        /// Number of raking elements per warp synchronous raking thread
-        SEGMENT_LENGTH = SHARING_THREADS / WARP_THREADS,
-    };
-
-    ///  WarpReduce utility type
-    typedef WarpReduce<T, RAKING_THREADS, PTX_ARCH> WarpReduce;
-
-    /// Layout type for padded thread block raking grid
-    typedef BlockRakingLayout<T, SHARING_THREADS, PTX_ARCH> BlockRakingLayout;
-
-    /// Shared memory storage layout type
-    union _TempStorage
-    {
-        struct
-        {
-            typename WarpReduce::TempStorage        warp_storage;        ///< Storage for warp-synchronous reduction
-            typename BlockRakingLayout::TempStorage raking_grid;         ///< Padded thread block raking grid
-        };
-        typename FallBack::TempStorage              fallback_storage;    ///< Fall-back storage for non-commutative block scan
-    };
-
-
-    /// Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    // Thread fields
-    _TempStorage &temp_storage;
-    unsigned int linear_tid;
-
-
-    /// Constructor
-    __device__ __forceinline__ BlockReduceRakingCommutativeOnly(
-        TempStorage &temp_storage)
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    /// Computes a thread block-wide reduction using addition (+) as the reduction operator. The first num_valid threads each contribute one reduction partial.  The return value is only valid for thread<sub>0</sub>.
-    template <bool FULL_TILE>
-    __device__ __forceinline__ T Sum(
-        T                   partial,            ///< [in] Calling thread's input partial reductions
-        int                 num_valid)          ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
-    {
-        if (USE_FALLBACK || !FULL_TILE)
-        {
-            return FallBack(temp_storage.fallback_storage).template Sum<FULL_TILE>(partial, num_valid);
-        }
-        else
-        {
-            // Place partial into shared memory grid
-            if (linear_tid >= RAKING_THREADS)
-                *BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid - RAKING_THREADS) = partial;
-
-            CTA_SYNC();
-
-            // Reduce parallelism to one warp
-            if (linear_tid < RAKING_THREADS)
-            {
-                // Raking reduction in grid
-                T *raking_segment = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid);
-                partial = internal::ThreadReduce<SEGMENT_LENGTH>(raking_segment, cub::Sum(), partial);
-
-                // Warpscan
-                partial = WarpReduce(temp_storage.warp_storage).Sum(partial);
-            }
-        }
-
-        return partial;
-    }
-
-
-    /// Computes a thread block-wide reduction using the specified reduction operator. The first num_valid threads each contribute one reduction partial.  The return value is only valid for thread<sub>0</sub>.
-    template <
-        bool                FULL_TILE,
-        typename            ReductionOp>
-    __device__ __forceinline__ T Reduce(
-        T                   partial,            ///< [in] Calling thread's input partial reductions
-        int                 num_valid,          ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
-        ReductionOp         reduction_op)       ///< [in] Binary reduction operator
-    {
-        if (USE_FALLBACK || !FULL_TILE)
-        {
-            return FallBack(temp_storage.fallback_storage).template Reduce<FULL_TILE>(partial, num_valid, reduction_op);
-        }
-        else
-        {
-            // Place partial into shared memory grid
-            if (linear_tid >= RAKING_THREADS)
-                *BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid - RAKING_THREADS) = partial;
-
-            CTA_SYNC();
-
-            // Reduce parallelism to one warp
-            if (linear_tid < RAKING_THREADS)
-            {
-                // Raking reduction in grid
-                T *raking_segment = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid);
-                partial = internal::ThreadReduce<SEGMENT_LENGTH>(raking_segment, reduction_op, partial);
-
-                // Warpscan
-                partial = WarpReduce(temp_storage.warp_storage).Reduce(partial, reduction_op);
-            }
-        }
-
-        return partial;
-    }
-
-};
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/block/specializations/block_reduce_warp_reductions.cuh b/pfsimulator/third_party/cub/block/specializations/block_reduce_warp_reductions.cuh
deleted file mode 100644
index 10ba303b4..000000000
--- a/pfsimulator/third_party/cub/block/specializations/block_reduce_warp_reductions.cuh
+++ /dev/null
@@ -1,218 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::BlockReduceWarpReductions provides variants of warp-reduction-based parallel reduction across a CUDA thread block.  Supports non-commutative reduction operators.
- */
-
-#pragma once
-
-#include "../../warp/warp_reduce.cuh"
-#include "../../util_ptx.cuh"
-#include "../../util_arch.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \brief BlockReduceWarpReductions provides variants of warp-reduction-based parallel reduction across a CUDA thread block.  Supports non-commutative reduction operators.
- */
-template <
-    typename    T,              ///< Data type being reduced
-    int         BLOCK_DIM_X,    ///< The thread block length in threads along the X dimension
-    int         BLOCK_DIM_Y,    ///< The thread block length in threads along the Y dimension
-    int         BLOCK_DIM_Z,    ///< The thread block length in threads along the Z dimension
-    int         PTX_ARCH>       ///< The PTX compute capability for which to to specialize this collective
-struct BlockReduceWarpReductions
-{
-    /// Constants
-    enum
-    {
-        /// The thread block size in threads
-        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-
-        /// Number of warp threads
-        WARP_THREADS = CUB_WARP_THREADS(PTX_ARCH),
-
-        /// Number of active warps
-        WARPS = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS,
-
-        /// The logical warp size for warp reductions
-        LOGICAL_WARP_SIZE = CUB_MIN(BLOCK_THREADS, WARP_THREADS),
-
-        /// Whether or not the logical warp size evenly divides the thread block size
-        EVEN_WARP_MULTIPLE = (BLOCK_THREADS % LOGICAL_WARP_SIZE == 0)
-    };
-
-
-    ///  WarpReduce utility type
-    typedef typename WarpReduce<T, LOGICAL_WARP_SIZE, PTX_ARCH>::InternalWarpReduce WarpReduce;
-
-
-    /// Shared memory storage layout type
-    struct _TempStorage
-    {
-        typename WarpReduce::TempStorage    warp_reduce[WARPS];         ///< Buffer for warp-synchronous scan
-        T                                   warp_aggregates[WARPS];     ///< Shared totals from each warp-synchronous scan
-        T                                   block_prefix;               ///< Shared prefix for the entire thread block
-    };
-
-    /// Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    // Thread fields
-    _TempStorage &temp_storage;
-    int linear_tid;
-    int warp_id;
-    int lane_id;
-
-
-    /// Constructor
-    __device__ __forceinline__ BlockReduceWarpReductions(
-        TempStorage &temp_storage)
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z)),
-        warp_id((WARPS == 1) ? 0 : linear_tid / WARP_THREADS),
-        lane_id(LaneId())
-    {}
-
-
-    template <bool FULL_TILE, typename ReductionOp, int SUCCESSOR_WARP>
-    __device__ __forceinline__ T ApplyWarpAggregates(
-        ReductionOp                 reduction_op,       ///< [in] Binary scan operator
-        T                           warp_aggregate,     ///< [in] <b>[<em>lane</em><sub>0</sub> only]</b> Warp-wide aggregate reduction of input items
-        int                         num_valid,          ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
-        Int2Type<SUCCESSOR_WARP>    /*successor_warp*/)
-    {
-        if (FULL_TILE || (SUCCESSOR_WARP * LOGICAL_WARP_SIZE < num_valid))
-        {
-            T addend = temp_storage.warp_aggregates[SUCCESSOR_WARP];
-            warp_aggregate = reduction_op(warp_aggregate, addend);
-        }
-        return ApplyWarpAggregates<FULL_TILE>(reduction_op, warp_aggregate, num_valid, Int2Type<SUCCESSOR_WARP + 1>());
-    }
-
-    template <bool FULL_TILE, typename ReductionOp>
-    __device__ __forceinline__ T ApplyWarpAggregates(
-        ReductionOp         /*reduction_op*/,   ///< [in] Binary scan operator
-        T                   warp_aggregate,     ///< [in] <b>[<em>lane</em><sub>0</sub> only]</b> Warp-wide aggregate reduction of input items
-        int                 /*num_valid*/,      ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
-        Int2Type<WARPS>     /*successor_warp*/)
-    {
-        return warp_aggregate;
-    }
-
-
-    /// Returns block-wide aggregate in <em>thread</em><sub>0</sub>.
-    template <
-        bool                FULL_TILE,
-        typename            ReductionOp>
-    __device__ __forceinline__ T ApplyWarpAggregates(
-        ReductionOp         reduction_op,       ///< [in] Binary scan operator
-        T                   warp_aggregate,     ///< [in] <b>[<em>lane</em><sub>0</sub> only]</b> Warp-wide aggregate reduction of input items
-        int                 num_valid)          ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
-    {
-        // Share lane aggregates
-        if (lane_id == 0)
-        {
-            temp_storage.warp_aggregates[warp_id] = warp_aggregate;
-        }
-
-        CTA_SYNC();
-
-        // Update total aggregate in warp 0, lane 0
-        if (linear_tid == 0)
-        {
-            warp_aggregate = ApplyWarpAggregates<FULL_TILE>(reduction_op, warp_aggregate, num_valid, Int2Type<1>());
-        }
-
-        return warp_aggregate;
-    }
-
-
-    /// Computes a thread block-wide reduction using addition (+) as the reduction operator. The first num_valid threads each contribute one reduction partial.  The return value is only valid for thread<sub>0</sub>.
-    template <bool FULL_TILE>
-    __device__ __forceinline__ T Sum(
-        T                   input,          ///< [in] Calling thread's input partial reductions
-        int                 num_valid)      ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
-    {
-        cub::Sum    reduction_op;
-        int         warp_offset = (warp_id * LOGICAL_WARP_SIZE);
-        int         warp_num_valid = ((FULL_TILE && EVEN_WARP_MULTIPLE) || (warp_offset + LOGICAL_WARP_SIZE <= num_valid)) ?
-                            LOGICAL_WARP_SIZE :
-                            num_valid - warp_offset;
-
-        // Warp reduction in every warp
-        T warp_aggregate = WarpReduce(temp_storage.warp_reduce[warp_id]).template Reduce<(FULL_TILE && EVEN_WARP_MULTIPLE)>(
-            input,
-            warp_num_valid,
-            cub::Sum());
-
-        // Update outputs and block_aggregate with warp-wide aggregates from lane-0s
-        return ApplyWarpAggregates<FULL_TILE>(reduction_op, warp_aggregate, num_valid);
-    }
-
-
-    /// Computes a thread block-wide reduction using the specified reduction operator. The first num_valid threads each contribute one reduction partial.  The return value is only valid for thread<sub>0</sub>.
-    template <
-        bool                FULL_TILE,
-        typename            ReductionOp>
-    __device__ __forceinline__ T Reduce(
-        T                   input,              ///< [in] Calling thread's input partial reductions
-        int                 num_valid,          ///< [in] Number of valid elements (may be less than BLOCK_THREADS)
-        ReductionOp         reduction_op)       ///< [in] Binary reduction operator
-    {
-        int         warp_offset = warp_id * LOGICAL_WARP_SIZE;
-        int         warp_num_valid = ((FULL_TILE && EVEN_WARP_MULTIPLE) || (warp_offset + LOGICAL_WARP_SIZE <= num_valid)) ?
-                            LOGICAL_WARP_SIZE :
-                            num_valid - warp_offset;
-
-        // Warp reduction in every warp
-        T warp_aggregate = WarpReduce(temp_storage.warp_reduce[warp_id]).template Reduce<(FULL_TILE && EVEN_WARP_MULTIPLE)>(
-            input,
-            warp_num_valid,
-            reduction_op);
-
-        // Update outputs and block_aggregate with warp-wide aggregates from lane-0s
-        return ApplyWarpAggregates<FULL_TILE>(reduction_op, warp_aggregate, num_valid);
-    }
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/block/specializations/block_scan_raking.cuh b/pfsimulator/third_party/cub/block/specializations/block_scan_raking.cuh
deleted file mode 100644
index a855cda0b..000000000
--- a/pfsimulator/third_party/cub/block/specializations/block_scan_raking.cuh
+++ /dev/null
@@ -1,666 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-
-/**
- * \file
- * cub::BlockScanRaking provides variants of raking-based parallel prefix scan across a CUDA thread block.
- */
-
-#pragma once
-
-#include "../../util_ptx.cuh"
-#include "../../util_arch.cuh"
-#include "../../block/block_raking_layout.cuh"
-#include "../../thread/thread_reduce.cuh"
-#include "../../thread/thread_scan.cuh"
-#include "../../warp/warp_scan.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \brief BlockScanRaking provides variants of raking-based parallel prefix scan across a CUDA thread block.
- */
-template <
-    typename    T,              ///< Data type being scanned
-    int         BLOCK_DIM_X,    ///< The thread block length in threads along the X dimension
-    int         BLOCK_DIM_Y,    ///< The thread block length in threads along the Y dimension
-    int         BLOCK_DIM_Z,    ///< The thread block length in threads along the Z dimension
-    bool        MEMOIZE,        ///< Whether or not to buffer outer raking scan partials to incur fewer shared memory reads at the expense of higher register pressure
-    int         PTX_ARCH>       ///< The PTX compute capability for which to to specialize this collective
-struct BlockScanRaking
-{
-    //---------------------------------------------------------------------
-    // Types and constants
-    //---------------------------------------------------------------------
-
-    /// Constants
-    enum
-    {
-        /// The thread block size in threads
-        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-    };
-
-    /// Layout type for padded thread block raking grid
-    typedef BlockRakingLayout<T, BLOCK_THREADS, PTX_ARCH> BlockRakingLayout;
-
-    /// Constants
-    enum
-    {
-        /// Number of raking threads
-        RAKING_THREADS = BlockRakingLayout::RAKING_THREADS,
-
-        /// Number of raking elements per warp synchronous raking thread
-        SEGMENT_LENGTH = BlockRakingLayout::SEGMENT_LENGTH,
-
-        /// Cooperative work can be entirely warp synchronous
-        WARP_SYNCHRONOUS = (BLOCK_THREADS == RAKING_THREADS),
-    };
-
-    ///  WarpScan utility type
-    typedef WarpScan<T, RAKING_THREADS, PTX_ARCH> WarpScan;
-
-    /// Shared memory storage layout type
-    struct _TempStorage
-    {
-        typename WarpScan::TempStorage              warp_scan;          ///< Buffer for warp-synchronous scan
-        typename BlockRakingLayout::TempStorage     raking_grid;        ///< Padded thread block raking grid
-        T                                           block_aggregate;    ///< Block aggregate
-    };
-
-
-    /// Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    //---------------------------------------------------------------------
-    // Per-thread fields
-    //---------------------------------------------------------------------
-
-    // Thread fields
-    _TempStorage    &temp_storage;
-    unsigned int    linear_tid;
-    T               cached_segment[SEGMENT_LENGTH];
-
-
-    //---------------------------------------------------------------------
-    // Utility methods
-    //---------------------------------------------------------------------
-
-    /// Templated reduction
-    template <int ITERATION, typename ScanOp>
-    __device__ __forceinline__ T GuardedReduce(
-        T*                  raking_ptr,         ///< [in] Input array
-        ScanOp              scan_op,            ///< [in] Binary reduction operator
-        T                   raking_partial,     ///< [in] Prefix to seed reduction with
-        Int2Type<ITERATION> /*iteration*/)
-    {
-        if ((BlockRakingLayout::UNGUARDED) || (((linear_tid * SEGMENT_LENGTH) + ITERATION) < BLOCK_THREADS))
-        {
-            T addend = raking_ptr[ITERATION];
-            raking_partial = scan_op(raking_partial, addend);
-        }
-
-        return GuardedReduce(raking_ptr, scan_op, raking_partial, Int2Type<ITERATION + 1>());
-    }
-
-
-    /// Templated reduction (base case)
-    template <typename ScanOp>
-    __device__ __forceinline__ T GuardedReduce(
-        T*                          /*raking_ptr*/,    ///< [in] Input array
-        ScanOp                      /*scan_op*/,       ///< [in] Binary reduction operator
-        T                           raking_partial,    ///< [in] Prefix to seed reduction with
-        Int2Type<SEGMENT_LENGTH>    /*iteration*/)
-    {
-        return raking_partial;
-    }
-
-
-    /// Templated copy
-    template <int ITERATION>
-    __device__ __forceinline__ void CopySegment(
-        T*                  out,            ///< [out] Out array
-        T*                  in,             ///< [in] Input array
-        Int2Type<ITERATION> /*iteration*/)
-    {
-        out[ITERATION] = in[ITERATION];
-        CopySegment(out, in, Int2Type<ITERATION + 1>());
-    }
-
- 
-    /// Templated copy (base case)
-    __device__ __forceinline__ void CopySegment(
-        T*                  /*out*/,            ///< [out] Out array
-        T*                  /*in*/,             ///< [in] Input array
-        Int2Type<SEGMENT_LENGTH> /*iteration*/)
-    {}
-
-
-    /// Performs upsweep raking reduction, returning the aggregate
-    template <typename ScanOp>
-    __device__ __forceinline__ T Upsweep(
-        ScanOp scan_op)
-    {
-        T *smem_raking_ptr = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid);
-
-        // Read data into registers
-        CopySegment(cached_segment, smem_raking_ptr, Int2Type<0>());
-
-        T raking_partial = cached_segment[0];
-
-        return GuardedReduce(cached_segment, scan_op, raking_partial, Int2Type<1>());
-    }
-
-
-    /// Performs exclusive downsweep raking scan
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveDownsweep(
-        ScanOp          scan_op,
-        T               raking_partial,
-        bool            apply_prefix = true)
-    {
-        T *smem_raking_ptr = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid);
-
-        // Read data back into registers
-        if (!MEMOIZE)
-        {
-            CopySegment(cached_segment, smem_raking_ptr, Int2Type<0>());
-        }
-
-        internal::ThreadScanExclusive(cached_segment, cached_segment, scan_op, raking_partial, apply_prefix);
-
-        // Write data back to smem
-        CopySegment(smem_raking_ptr, cached_segment, Int2Type<0>());
-    }
-
-
-    /// Performs inclusive downsweep raking scan
-    template <typename ScanOp>
-    __device__ __forceinline__ void InclusiveDownsweep(
-        ScanOp          scan_op,
-        T               raking_partial,
-        bool            apply_prefix = true)
-    {
-        T *smem_raking_ptr = BlockRakingLayout::RakingPtr(temp_storage.raking_grid, linear_tid);
-
-        // Read data back into registers
-        if (!MEMOIZE)
-        {
-            CopySegment(cached_segment, smem_raking_ptr, Int2Type<0>());
-        }
-
-        internal::ThreadScanInclusive(cached_segment, cached_segment, scan_op, raking_partial, apply_prefix);
-
-        // Write data back to smem
-        CopySegment(smem_raking_ptr, cached_segment, Int2Type<0>());
-    }
-
-
-    //---------------------------------------------------------------------
-    // Constructors
-    //---------------------------------------------------------------------
-
-    /// Constructor
-    __device__ __forceinline__ BlockScanRaking(
-        TempStorage &temp_storage)
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
-    {}
-
-
-    //---------------------------------------------------------------------
-    // Exclusive scans
-    //---------------------------------------------------------------------
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &exclusive_output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op)                        ///< [in] Binary scan operator
-    {
-        if (WARP_SYNCHRONOUS)
-        {
-            // Short-circuit directly to warp-synchronous scan
-            WarpScan(temp_storage.warp_scan).ExclusiveScan(input, exclusive_output, scan_op);
-        }
-        else
-        {
-            // Place thread partial into shared memory raking grid
-            T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid);
-            *placement_ptr = input;
-
-            CTA_SYNC();
-
-            // Reduce parallelism down to just raking threads
-            if (linear_tid < RAKING_THREADS)
-            {
-                // Raking upsweep reduction across shared partials
-                T upsweep_partial = Upsweep(scan_op);
-
-                // Warp-synchronous scan
-                T exclusive_partial;
-                WarpScan(temp_storage.warp_scan).ExclusiveScan(upsweep_partial, exclusive_partial, scan_op);
-
-                // Exclusive raking downsweep scan
-                ExclusiveDownsweep(scan_op, exclusive_partial, (linear_tid != 0));
-            }
-
-            CTA_SYNC();
-
-            // Grab thread prefix from shared memory
-            exclusive_output = *placement_ptr;
-        }
-    }
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,              ///< [in] Calling thread's input items
-        T               &output,            ///< [out] Calling thread's output items (may be aliased to \p input)
-        const T         &initial_value,     ///< [in] Initial value to seed the exclusive scan
-        ScanOp          scan_op)            ///< [in] Binary scan operator
-    {
-        if (WARP_SYNCHRONOUS)
-        {
-            // Short-circuit directly to warp-synchronous scan
-            WarpScan(temp_storage.warp_scan).ExclusiveScan(input, output, initial_value, scan_op);
-        }
-        else
-        {
-            // Place thread partial into shared memory raking grid
-            T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid);
-            *placement_ptr = input;
-
-            CTA_SYNC();
-
-            // Reduce parallelism down to just raking threads
-            if (linear_tid < RAKING_THREADS)
-            {
-                // Raking upsweep reduction across shared partials
-                T upsweep_partial = Upsweep(scan_op);
-
-                // Exclusive Warp-synchronous scan
-                T exclusive_partial;
-                WarpScan(temp_storage.warp_scan).ExclusiveScan(upsweep_partial, exclusive_partial, initial_value, scan_op);
-
-                // Exclusive raking downsweep scan
-                ExclusiveDownsweep(scan_op, exclusive_partial);
-            }
-
-            CTA_SYNC();
-
-            // Grab exclusive partial from shared memory
-            output = *placement_ptr;
-        }
-    }
-
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op,                        ///< [in] Binary scan operator
-        T               &block_aggregate)               ///< [out] Threadblock-wide aggregate reduction of input items
-    {
-        if (WARP_SYNCHRONOUS)
-        {
-            // Short-circuit directly to warp-synchronous scan
-            WarpScan(temp_storage.warp_scan).ExclusiveScan(input, output, scan_op, block_aggregate);
-        }
-        else
-        {
-            // Place thread partial into shared memory raking grid
-            T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid);
-            *placement_ptr = input;
-
-            CTA_SYNC();
-
-            // Reduce parallelism down to just raking threads
-            if (linear_tid < RAKING_THREADS)
-            {
-                // Raking upsweep reduction across shared partials
-                T upsweep_partial= Upsweep(scan_op);
-
-                // Warp-synchronous scan
-                T inclusive_partial;
-                T exclusive_partial;
-                WarpScan(temp_storage.warp_scan).Scan(upsweep_partial, inclusive_partial, exclusive_partial, scan_op);
-
-                // Exclusive raking downsweep scan
-                ExclusiveDownsweep(scan_op, exclusive_partial, (linear_tid != 0));
-
-                // Broadcast aggregate to all threads
-                if (linear_tid == RAKING_THREADS - 1)
-                    temp_storage.block_aggregate = inclusive_partial;
-            }
-
-            CTA_SYNC();
-
-            // Grab thread prefix from shared memory
-            output = *placement_ptr;
-
-            // Retrieve block aggregate
-            block_aggregate = temp_storage.block_aggregate;
-        }
-    }
-
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,              ///< [in] Calling thread's input items
-        T               &output,            ///< [out] Calling thread's output items (may be aliased to \p input)
-        const T         &initial_value,     ///< [in] Initial value to seed the exclusive scan
-        ScanOp          scan_op,            ///< [in] Binary scan operator
-        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
-    {
-        if (WARP_SYNCHRONOUS)
-        {
-            // Short-circuit directly to warp-synchronous scan
-            WarpScan(temp_storage.warp_scan).ExclusiveScan(input, output, initial_value, scan_op, block_aggregate);
-        }
-        else
-        {
-            // Place thread partial into shared memory raking grid
-            T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid);
-            *placement_ptr = input;
-
-            CTA_SYNC();
-
-            // Reduce parallelism down to just raking threads
-            if (linear_tid < RAKING_THREADS)
-            {
-                // Raking upsweep reduction across shared partials
-                T upsweep_partial = Upsweep(scan_op);
-
-                // Warp-synchronous scan
-                T exclusive_partial;
-                WarpScan(temp_storage.warp_scan).ExclusiveScan(upsweep_partial, exclusive_partial, initial_value, scan_op, block_aggregate);
-
-                // Exclusive raking downsweep scan
-                ExclusiveDownsweep(scan_op, exclusive_partial);
-
-                // Broadcast aggregate to other threads
-                if (linear_tid == 0)
-                    temp_storage.block_aggregate = block_aggregate;
-            }
-
-            CTA_SYNC();
-
-            // Grab exclusive partial from shared memory
-            output = *placement_ptr;
-
-            // Retrieve block aggregate
-            block_aggregate = temp_storage.block_aggregate;
-        }
-    }
-
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-    template <
-        typename ScanOp,
-        typename BlockPrefixCallbackOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T                       input,                          ///< [in] Calling thread's input item
-        T                       &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp                  scan_op,                        ///< [in] Binary scan operator
-        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a thread block-wide prefix to be applied to all inputs.
-    {
-        if (WARP_SYNCHRONOUS)
-        {
-            // Short-circuit directly to warp-synchronous scan
-            T block_aggregate;
-            WarpScan warp_scan(temp_storage.warp_scan);
-            warp_scan.ExclusiveScan(input, output, scan_op, block_aggregate);
-
-            // Obtain warp-wide prefix in lane0, then broadcast to other lanes
-            T block_prefix = block_prefix_callback_op(block_aggregate);
-            block_prefix = warp_scan.Broadcast(block_prefix, 0);
-
-            output = scan_op(block_prefix, output);
-            if (linear_tid == 0)
-                output = block_prefix;
-        }
-        else
-        {
-            // Place thread partial into shared memory raking grid
-            T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid);
-            *placement_ptr = input;
-
-            CTA_SYNC();
-
-            // Reduce parallelism down to just raking threads
-            if (linear_tid < RAKING_THREADS)
-            {
-                WarpScan warp_scan(temp_storage.warp_scan);
-
-                // Raking upsweep reduction across shared partials
-                T upsweep_partial = Upsweep(scan_op);
-
-                // Warp-synchronous scan
-                T exclusive_partial, block_aggregate;
-                warp_scan.ExclusiveScan(upsweep_partial, exclusive_partial, scan_op, block_aggregate);
-
-                // Obtain block-wide prefix in lane0, then broadcast to other lanes
-                T block_prefix = block_prefix_callback_op(block_aggregate);
-                block_prefix = warp_scan.Broadcast(block_prefix, 0);
-
-                // Update prefix with warpscan exclusive partial
-                T downsweep_prefix = scan_op(block_prefix, exclusive_partial);
-                if (linear_tid == 0)
-                    downsweep_prefix = block_prefix;
-
-                // Exclusive raking downsweep scan
-                ExclusiveDownsweep(scan_op, downsweep_prefix);
-            }
-
-            CTA_SYNC();
-
-            // Grab thread prefix from shared memory
-            output = *placement_ptr;
-        }
-    }
-
-
-    //---------------------------------------------------------------------
-    // Inclusive scans
-    //---------------------------------------------------------------------
-
-    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
-    template <typename ScanOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op)                        ///< [in] Binary scan operator
-    {
-        if (WARP_SYNCHRONOUS)
-        {
-            // Short-circuit directly to warp-synchronous scan
-            WarpScan(temp_storage.warp_scan).InclusiveScan(input, output, scan_op);
-        }
-        else
-        {
-            // Place thread partial into shared memory raking grid
-            T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid);
-            *placement_ptr = input;
-
-            CTA_SYNC();
-
-            // Reduce parallelism down to just raking threads
-            if (linear_tid < RAKING_THREADS)
-            {
-                // Raking upsweep reduction across shared partials
-                T upsweep_partial = Upsweep(scan_op);
-
-                // Exclusive Warp-synchronous scan
-                T exclusive_partial;
-                WarpScan(temp_storage.warp_scan).ExclusiveScan(upsweep_partial, exclusive_partial, scan_op);
-
-                // Inclusive raking downsweep scan
-                InclusiveDownsweep(scan_op, exclusive_partial, (linear_tid != 0));
-            }
-
-            CTA_SYNC();
-
-            // Grab thread prefix from shared memory
-            output = *placement_ptr;
-        }
-    }
-
-
-    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-    template <typename ScanOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op,                        ///< [in] Binary scan operator
-        T               &block_aggregate)               ///< [out] Threadblock-wide aggregate reduction of input items
-    {
-        if (WARP_SYNCHRONOUS)
-        {
-            // Short-circuit directly to warp-synchronous scan
-            WarpScan(temp_storage.warp_scan).InclusiveScan(input, output, scan_op, block_aggregate);
-        }
-        else
-        {
-            // Place thread partial into shared memory raking grid
-            T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid);
-            *placement_ptr = input;
-
-            CTA_SYNC();
-
-            // Reduce parallelism down to just raking threads
-            if (linear_tid < RAKING_THREADS)
-            {
-                // Raking upsweep reduction across shared partials
-                T upsweep_partial = Upsweep(scan_op);
-
-                // Warp-synchronous scan
-                T inclusive_partial;
-                T exclusive_partial;
-                WarpScan(temp_storage.warp_scan).Scan(upsweep_partial, inclusive_partial, exclusive_partial, scan_op);
-
-                // Inclusive raking downsweep scan
-                InclusiveDownsweep(scan_op, exclusive_partial, (linear_tid != 0));
-
-                // Broadcast aggregate to all threads
-                if (linear_tid == RAKING_THREADS - 1)
-                    temp_storage.block_aggregate = inclusive_partial;
-            }
-
-            CTA_SYNC();
-
-            // Grab thread prefix from shared memory
-            output = *placement_ptr;
-
-            // Retrieve block aggregate
-            block_aggregate = temp_storage.block_aggregate;
-        }
-    }
-
-
-    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-    template <
-        typename ScanOp,
-        typename BlockPrefixCallbackOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T                       input,                          ///< [in] Calling thread's input item
-        T                       &output,                        ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp                  scan_op,                        ///< [in] Binary scan operator
-        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a thread block-wide prefix to be applied to all inputs.
-    {
-        if (WARP_SYNCHRONOUS)
-        {
-            // Short-circuit directly to warp-synchronous scan
-            T block_aggregate;
-            WarpScan warp_scan(temp_storage.warp_scan);
-            warp_scan.InclusiveScan(input, output, scan_op, block_aggregate);
-
-            // Obtain warp-wide prefix in lane0, then broadcast to other lanes
-            T block_prefix = block_prefix_callback_op(block_aggregate);
-            block_prefix = warp_scan.Broadcast(block_prefix, 0);
-
-            // Update prefix with exclusive warpscan partial
-            output = scan_op(block_prefix, output);
-        }
-        else
-        {
-            // Place thread partial into shared memory raking grid
-            T *placement_ptr = BlockRakingLayout::PlacementPtr(temp_storage.raking_grid, linear_tid);
-            *placement_ptr = input;
-
-            CTA_SYNC();
-
-            // Reduce parallelism down to just raking threads
-            if (linear_tid < RAKING_THREADS)
-            {
-                WarpScan warp_scan(temp_storage.warp_scan);
-
-                // Raking upsweep reduction across shared partials
-                T upsweep_partial = Upsweep(scan_op);
-
-                // Warp-synchronous scan
-                T exclusive_partial, block_aggregate;
-                warp_scan.ExclusiveScan(upsweep_partial, exclusive_partial, scan_op, block_aggregate);
-
-                // Obtain block-wide prefix in lane0, then broadcast to other lanes
-                T block_prefix = block_prefix_callback_op(block_aggregate);
-                block_prefix = warp_scan.Broadcast(block_prefix, 0);
-
-                // Update prefix with warpscan exclusive partial
-                T downsweep_prefix = scan_op(block_prefix, exclusive_partial);
-                if (linear_tid == 0)
-                    downsweep_prefix = block_prefix;
-
-                // Inclusive raking downsweep scan
-                InclusiveDownsweep(scan_op, downsweep_prefix);
-            }
-
-            CTA_SYNC();
-
-            // Grab thread prefix from shared memory
-            output = *placement_ptr;
-        }
-    }
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/block/specializations/block_scan_warp_scans.cuh b/pfsimulator/third_party/cub/block/specializations/block_scan_warp_scans.cuh
deleted file mode 100644
index 85e4d6135..000000000
--- a/pfsimulator/third_party/cub/block/specializations/block_scan_warp_scans.cuh
+++ /dev/null
@@ -1,392 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::BlockScanWarpscans provides warpscan-based variants of parallel prefix scan across a CUDA thread block.
- */
-
-#pragma once
-
-#include "../../util_arch.cuh"
-#include "../../util_ptx.cuh"
-#include "../../warp/warp_scan.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \brief BlockScanWarpScans provides warpscan-based variants of parallel prefix scan across a CUDA thread block.
- */
-template <
-    typename    T,
-    int         BLOCK_DIM_X,    ///< The thread block length in threads along the X dimension
-    int         BLOCK_DIM_Y,    ///< The thread block length in threads along the Y dimension
-    int         BLOCK_DIM_Z,    ///< The thread block length in threads along the Z dimension
-    int         PTX_ARCH>       ///< The PTX compute capability for which to to specialize this collective
-struct BlockScanWarpScans
-{
-    //---------------------------------------------------------------------
-    // Types and constants
-    //---------------------------------------------------------------------
-
-    /// Constants
-    enum
-    {
-        /// Number of warp threads
-        WARP_THREADS = CUB_WARP_THREADS(PTX_ARCH),
-
-        /// The thread block size in threads
-        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-
-        /// Number of active warps
-        WARPS = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS,
-    };
-
-    ///  WarpScan utility type
-    typedef WarpScan<T, WARP_THREADS, PTX_ARCH> WarpScanT;
-
-    ///  WarpScan utility type
-    typedef WarpScan<T, WARPS, PTX_ARCH> WarpAggregateScan;
-
-    /// Shared memory storage layout type
-
-    struct __align__(32) _TempStorage
-    {
-        T                               warp_aggregates[WARPS];
-        typename WarpScanT::TempStorage warp_scan[WARPS];           ///< Buffer for warp-synchronous scans
-        T                               block_prefix;               ///< Shared prefix for the entire thread block
-    };
-
-
-    /// Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    //---------------------------------------------------------------------
-    // Per-thread fields
-    //---------------------------------------------------------------------
-
-    // Thread fields
-    _TempStorage    &temp_storage;
-    unsigned int    linear_tid;
-    unsigned int    warp_id;
-    unsigned int    lane_id;
-
-
-    //---------------------------------------------------------------------
-    // Constructors
-    //---------------------------------------------------------------------
-
-    /// Constructor
-    __device__ __forceinline__ BlockScanWarpScans(
-        TempStorage &temp_storage)
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z)),
-        warp_id((WARPS == 1) ? 0 : linear_tid / WARP_THREADS),
-        lane_id(LaneId())
-    {}
-
-
-    //---------------------------------------------------------------------
-    // Utility methods
-    //---------------------------------------------------------------------
-
-    template <typename ScanOp, int WARP>
-    __device__ __forceinline__ void ApplyWarpAggregates(
-        T               &warp_prefix,           ///< [out] The calling thread's partial reduction
-        ScanOp          scan_op,            ///< [in] Binary scan operator
-        T               &block_aggregate,   ///< [out] Threadblock-wide aggregate reduction of input items
-        Int2Type<WARP>  /*addend_warp*/)
-    {
-        if (warp_id == WARP)
-            warp_prefix = block_aggregate;
-
-        T addend = temp_storage.warp_aggregates[WARP];
-        block_aggregate = scan_op(block_aggregate, addend);
-
-        ApplyWarpAggregates(warp_prefix, scan_op, block_aggregate, Int2Type<WARP + 1>());
-    }
-
-    template <typename ScanOp>
-    __device__ __forceinline__ void ApplyWarpAggregates(
-        T               &/*warp_prefix*/,       ///< [out] The calling thread's partial reduction
-        ScanOp          /*scan_op*/,            ///< [in] Binary scan operator
-        T               &/*block_aggregate*/,   ///< [out] Threadblock-wide aggregate reduction of input items
-        Int2Type<WARPS> /*addend_warp*/)
-    {}
-
-
-    /// Use the warp-wide aggregates to compute the calling warp's prefix.  Also returns block-wide aggregate in all threads.
-    template <typename ScanOp>
-    __device__ __forceinline__ T ComputeWarpPrefix(
-        ScanOp          scan_op,            ///< [in] Binary scan operator
-        T               warp_aggregate,     ///< [in] <b>[<em>lane</em><sub>WARP_THREADS - 1</sub> only]</b> Warp-wide aggregate reduction of input items
-        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
-    {
-        // Last lane in each warp shares its warp-aggregate
-        if (lane_id == WARP_THREADS - 1)
-            temp_storage.warp_aggregates[warp_id] = warp_aggregate;
-
-        CTA_SYNC();
-
-        // Accumulate block aggregates and save the one that is our warp's prefix
-        T warp_prefix;
-        block_aggregate = temp_storage.warp_aggregates[0];
-
-        // Use template unrolling (since the PTX backend can't handle unrolling it for SM1x)
-        ApplyWarpAggregates(warp_prefix, scan_op, block_aggregate, Int2Type<1>());
-/*
-        #pragma unroll
-        for (int WARP = 1; WARP < WARPS; ++WARP)
-        {
-            if (warp_id == WARP)
-                warp_prefix = block_aggregate;
-
-            T addend = temp_storage.warp_aggregates[WARP];
-            block_aggregate = scan_op(block_aggregate, addend);
-        }
-*/
-
-        return warp_prefix;
-    }
-
-
-    /// Use the warp-wide aggregates and initial-value to compute the calling warp's prefix.  Also returns block-wide aggregate in all threads.
-    template <typename ScanOp>
-    __device__ __forceinline__ T ComputeWarpPrefix(
-        ScanOp          scan_op,            ///< [in] Binary scan operator
-        T               warp_aggregate,     ///< [in] <b>[<em>lane</em><sub>WARP_THREADS - 1</sub> only]</b> Warp-wide aggregate reduction of input items
-        T               &block_aggregate,   ///< [out] Threadblock-wide aggregate reduction of input items
-        const T         &initial_value)     ///< [in] Initial value to seed the exclusive scan
-    {
-        T warp_prefix = ComputeWarpPrefix(scan_op, warp_aggregate, block_aggregate);
-
-        warp_prefix = scan_op(initial_value, warp_prefix);
-
-        if (warp_id == 0)
-            warp_prefix = initial_value;
-
-        return warp_prefix;
-    }
-
-    //---------------------------------------------------------------------
-    // Exclusive scans
-    //---------------------------------------------------------------------
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &exclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op)                        ///< [in] Binary scan operator
-    {
-        // Compute block-wide exclusive scan.  The exclusive output from tid0 is invalid.
-        T block_aggregate;
-        ExclusiveScan(input, exclusive_output, scan_op, block_aggregate);
-    }
-
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,              ///< [in] Calling thread's input items
-        T               &exclusive_output,  ///< [out] Calling thread's output items (may be aliased to \p input)
-        const T         &initial_value,     ///< [in] Initial value to seed the exclusive scan
-        ScanOp          scan_op)            ///< [in] Binary scan operator
-    {
-        T block_aggregate;
-        ExclusiveScan(input, exclusive_output, initial_value, scan_op, block_aggregate);
-    }
-
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,              ///< [in] Calling thread's input item
-        T               &exclusive_output,  ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op,            ///< [in] Binary scan operator
-        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
-    {
-        // Compute warp scan in each warp.  The exclusive output from each lane0 is invalid.
-        T inclusive_output;
-        WarpScanT(temp_storage.warp_scan[warp_id]).Scan(input, inclusive_output, exclusive_output, scan_op);
-
-        // Compute the warp-wide prefix and block-wide aggregate for each warp.  Warp prefix for warp0 is invalid.
-        T warp_prefix = ComputeWarpPrefix(scan_op, inclusive_output, block_aggregate);
-
-        // Apply warp prefix to our lane's partial
-        if (warp_id != 0)
-        {
-            exclusive_output = scan_op(warp_prefix, exclusive_output);
-            if (lane_id == 0)
-                exclusive_output = warp_prefix;
-        }
-    }
-
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,              ///< [in] Calling thread's input items
-        T               &exclusive_output,  ///< [out] Calling thread's output items (may be aliased to \p input)
-        const T         &initial_value,     ///< [in] Initial value to seed the exclusive scan
-        ScanOp          scan_op,            ///< [in] Binary scan operator
-        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
-    {
-        // Compute warp scan in each warp.  The exclusive output from each lane0 is invalid.
-        T inclusive_output;
-        WarpScanT(temp_storage.warp_scan[warp_id]).Scan(input, inclusive_output, exclusive_output, scan_op);
-
-        // Compute the warp-wide prefix and block-wide aggregate for each warp
-        T warp_prefix = ComputeWarpPrefix(scan_op, inclusive_output, block_aggregate, initial_value);
-
-        // Apply warp prefix to our lane's partial
-        exclusive_output = scan_op(warp_prefix, exclusive_output);
-        if (lane_id == 0)
-            exclusive_output = warp_prefix;
-    }
-
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-    template <
-        typename ScanOp,
-        typename BlockPrefixCallbackOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T                       input,                          ///< [in] Calling thread's input item
-        T                       &exclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp                  scan_op,                        ///< [in] Binary scan operator
-        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a thread block-wide prefix to be applied to all inputs.
-    {
-        // Compute block-wide exclusive scan.  The exclusive output from tid0 is invalid.
-        T block_aggregate;
-        ExclusiveScan(input, exclusive_output, scan_op, block_aggregate);
-
-        // Use the first warp to determine the thread block prefix, returning the result in lane0
-        if (warp_id == 0)
-        {
-            T block_prefix = block_prefix_callback_op(block_aggregate);
-            if (lane_id == 0)
-            {
-                // Share the prefix with all threads
-                temp_storage.block_prefix = block_prefix;
-                exclusive_output = block_prefix;                // The block prefix is the exclusive output for tid0
-            }
-        }
-
-        CTA_SYNC();
-
-        // Incorporate thread block prefix into outputs
-        T block_prefix = temp_storage.block_prefix;
-        if (linear_tid > 0)
-        {
-            exclusive_output = scan_op(block_prefix, exclusive_output);
-        }
-    }
-
-
-    //---------------------------------------------------------------------
-    // Inclusive scans
-    //---------------------------------------------------------------------
-
-    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
-    template <typename ScanOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &inclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op)                        ///< [in] Binary scan operator
-    {
-        T block_aggregate;
-        InclusiveScan(input, inclusive_output, scan_op, block_aggregate);
-    }
-
-
-    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-    template <typename ScanOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &inclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op,                        ///< [in] Binary scan operator
-        T               &block_aggregate)               ///< [out] Threadblock-wide aggregate reduction of input items
-    {
-        WarpScanT(temp_storage.warp_scan[warp_id]).InclusiveScan(input, inclusive_output, scan_op);
-
-        // Compute the warp-wide prefix and block-wide aggregate for each warp.  Warp prefix for warp0 is invalid.
-        T warp_prefix = ComputeWarpPrefix(scan_op, inclusive_output, block_aggregate);
-
-        // Apply warp prefix to our lane's partial
-        if (warp_id != 0)
-        {
-            inclusive_output = scan_op(warp_prefix, inclusive_output);
-        }
-    }
-
-
-    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-    template <
-        typename ScanOp,
-        typename BlockPrefixCallbackOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T                       input,                          ///< [in] Calling thread's input item
-        T                       &exclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp                  scan_op,                        ///< [in] Binary scan operator
-        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a thread block-wide prefix to be applied to all inputs.
-    {
-        T block_aggregate;
-        InclusiveScan(input, exclusive_output, scan_op, block_aggregate);
-
-        // Use the first warp to determine the thread block prefix, returning the result in lane0
-        if (warp_id == 0)
-        {
-            T block_prefix = block_prefix_callback_op(block_aggregate);
-            if (lane_id == 0)
-            {
-                // Share the prefix with all threads
-                temp_storage.block_prefix = block_prefix;
-            }
-        }
-
-        CTA_SYNC();
-
-        // Incorporate thread block prefix into outputs
-        T block_prefix = temp_storage.block_prefix;
-        exclusive_output = scan_op(block_prefix, exclusive_output);
-    }
-
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/block/specializations/block_scan_warp_scans2.cuh b/pfsimulator/third_party/cub/block/specializations/block_scan_warp_scans2.cuh
deleted file mode 100644
index 4de7c69b7..000000000
--- a/pfsimulator/third_party/cub/block/specializations/block_scan_warp_scans2.cuh
+++ /dev/null
@@ -1,436 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::BlockScanWarpscans provides warpscan-based variants of parallel prefix scan across a CUDA thread block.
- */
-
-#pragma once
-
-#include "../../util_arch.cuh"
-#include "../../util_ptx.cuh"
-#include "../../warp/warp_scan.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \brief BlockScanWarpScans provides warpscan-based variants of parallel prefix scan across a CUDA thread block.
- */
-template <
-    typename    T,
-    int         BLOCK_DIM_X,    ///< The thread block length in threads along the X dimension
-    int         BLOCK_DIM_Y,    ///< The thread block length in threads along the Y dimension
-    int         BLOCK_DIM_Z,    ///< The thread block length in threads along the Z dimension
-    int         PTX_ARCH>       ///< The PTX compute capability for which to to specialize this collective
-struct BlockScanWarpScans
-{
-    //---------------------------------------------------------------------
-    // Types and constants
-    //---------------------------------------------------------------------
-
-    /// Constants
-    enum
-    {
-        /// Number of warp threads
-        WARP_THREADS = CUB_WARP_THREADS(PTX_ARCH),
-
-        /// The thread block size in threads
-        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-
-        /// Number of active warps
-        WARPS = (BLOCK_THREADS + WARP_THREADS - 1) / WARP_THREADS,
-    };
-
-    ///  WarpScan utility type
-    typedef WarpScan<T, WARP_THREADS, PTX_ARCH> WarpScanT;
-
-    ///  WarpScan utility type
-    typedef WarpScan<T, WARPS, PTX_ARCH> WarpAggregateScanT;
-
-    /// Shared memory storage layout type
-    struct _TempStorage
-    {
-        typename WarpAggregateScanT::TempStorage    inner_scan[WARPS];          ///< Buffer for warp-synchronous scans
-        typename WarpScanT::TempStorage             warp_scan[WARPS];           ///< Buffer for warp-synchronous scans
-        T                                           warp_aggregates[WARPS];
-        T                                           block_prefix;               ///< Shared prefix for the entire thread block
-    };
-
-
-    /// Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    //---------------------------------------------------------------------
-    // Per-thread fields
-    //---------------------------------------------------------------------
-
-    // Thread fields
-    _TempStorage    &temp_storage;
-    unsigned int    linear_tid;
-    unsigned int    warp_id;
-    unsigned int    lane_id;
-
-
-    //---------------------------------------------------------------------
-    // Constructors
-    //---------------------------------------------------------------------
-
-    /// Constructor
-    __device__ __forceinline__ BlockScanWarpScans(
-        TempStorage &temp_storage)
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z)),
-        warp_id((WARPS == 1) ? 0 : linear_tid / WARP_THREADS),
-        lane_id(LaneId())
-    {}
-
-
-    //---------------------------------------------------------------------
-    // Utility methods
-    //---------------------------------------------------------------------
-
-    template <typename ScanOp, int WARP>
-    __device__ __forceinline__ void ApplyWarpAggregates(
-        T               &warp_prefix,           ///< [out] The calling thread's partial reduction
-        ScanOp          scan_op,            ///< [in] Binary scan operator
-        T               &block_aggregate,   ///< [out] Threadblock-wide aggregate reduction of input items
-        Int2Type<WARP>  addend_warp)
-    {
-        if (warp_id == WARP)
-            warp_prefix = block_aggregate;
-
-        T addend = temp_storage.warp_aggregates[WARP];
-        block_aggregate = scan_op(block_aggregate, addend);
-
-        ApplyWarpAggregates(warp_prefix, scan_op, block_aggregate, Int2Type<WARP + 1>());
-    }
-
-    template <typename ScanOp>
-    __device__ __forceinline__ void ApplyWarpAggregates(
-        T               &warp_prefix,           ///< [out] The calling thread's partial reduction
-        ScanOp          scan_op,            ///< [in] Binary scan operator
-        T               &block_aggregate,   ///< [out] Threadblock-wide aggregate reduction of input items
-        Int2Type<WARPS> addend_warp)
-    {}
-
-
-    /// Use the warp-wide aggregates to compute the calling warp's prefix.  Also returns block-wide aggregate in all threads.
-    template <typename ScanOp>
-    __device__ __forceinline__ T ComputeWarpPrefix(
-        ScanOp          scan_op,            ///< [in] Binary scan operator
-        T               warp_aggregate,     ///< [in] <b>[<em>lane</em><sub>WARP_THREADS - 1</sub> only]</b> Warp-wide aggregate reduction of input items
-        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
-    {
-        // Last lane in each warp shares its warp-aggregate
-        if (lane_id == WARP_THREADS - 1)
-            temp_storage.warp_aggregates[warp_id] = warp_aggregate;
-
-        CTA_SYNC();
-
-        // Accumulate block aggregates and save the one that is our warp's prefix
-        T warp_prefix;
-        block_aggregate = temp_storage.warp_aggregates[0];
-
-        // Use template unrolling (since the PTX backend can't handle unrolling it for SM1x)
-        ApplyWarpAggregates(warp_prefix, scan_op, block_aggregate, Int2Type<1>());
-/*
-        #pragma unroll
-        for (int WARP = 1; WARP < WARPS; ++WARP)
-        {
-            if (warp_id == WARP)
-                warp_prefix = block_aggregate;
-
-            T addend = temp_storage.warp_aggregates[WARP];
-            block_aggregate = scan_op(block_aggregate, addend);
-        }
-*/
-
-        return warp_prefix;
-    }
-
-
-    /// Use the warp-wide aggregates and initial-value to compute the calling warp's prefix.  Also returns block-wide aggregate in all threads.
-    template <typename ScanOp>
-    __device__ __forceinline__ T ComputeWarpPrefix(
-        ScanOp          scan_op,            ///< [in] Binary scan operator
-        T               warp_aggregate,     ///< [in] <b>[<em>lane</em><sub>WARP_THREADS - 1</sub> only]</b> Warp-wide aggregate reduction of input items
-        T               &block_aggregate,   ///< [out] Threadblock-wide aggregate reduction of input items
-        const T         &initial_value)     ///< [in] Initial value to seed the exclusive scan
-    {
-        T warp_prefix = ComputeWarpPrefix(scan_op, warp_aggregate, block_aggregate);
-
-        warp_prefix = scan_op(initial_value, warp_prefix);
-
-        if (warp_id == 0)
-            warp_prefix = initial_value;
-
-        return warp_prefix;
-    }
-
-    //---------------------------------------------------------------------
-    // Exclusive scans
-    //---------------------------------------------------------------------
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &exclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op)                        ///< [in] Binary scan operator
-    {
-        // Compute block-wide exclusive scan.  The exclusive output from tid0 is invalid.
-        T block_aggregate;
-        ExclusiveScan(input, exclusive_output, scan_op, block_aggregate);
-    }
-
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,              ///< [in] Calling thread's input items
-        T               &exclusive_output,  ///< [out] Calling thread's output items (may be aliased to \p input)
-        const T         &initial_value,     ///< [in] Initial value to seed the exclusive scan
-        ScanOp          scan_op)            ///< [in] Binary scan operator
-    {
-        T block_aggregate;
-        ExclusiveScan(input, exclusive_output, initial_value, scan_op, block_aggregate);
-    }
-
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,              ///< [in] Calling thread's input item
-        T               &exclusive_output,  ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op,            ///< [in] Binary scan operator
-        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
-    {
-        WarpScanT my_warp_scan(temp_storage.warp_scan[warp_id]);
-
-        // Compute warp scan in each warp.  The exclusive output from each lane0 is invalid.
-        T inclusive_output;
-        my_warp_scan.Scan(input, inclusive_output, exclusive_output, scan_op);
-
-        // Compute the warp-wide prefix and block-wide aggregate for each warp.  Warp prefix for warp0 is invalid.
-//        T warp_prefix = ComputeWarpPrefix(scan_op, inclusive_output, block_aggregate);
-
-//--------------------------------------------------
-        // Last lane in each warp shares its warp-aggregate
-        if (lane_id == WARP_THREADS - 1)
-            temp_storage.warp_aggregates[warp_id] = inclusive_output;
-
-        CTA_SYNC();
-
-        // Get the warp scan partial
-        T warp_inclusive, warp_prefix;
-        if (lane_id < WARPS)
-        {
-            // Scan the warpscan partials
-            T warp_val = temp_storage.warp_aggregates[lane_id];
-            WarpAggregateScanT(temp_storage.inner_scan[warp_id]).Scan(warp_val, warp_inclusive, warp_prefix, scan_op);
-        }
-
-        warp_prefix         = my_warp_scan.Broadcast(warp_prefix, warp_id);
-        block_aggregate     = my_warp_scan.Broadcast(warp_inclusive, WARPS - 1);
-//--------------------------------------------------
-
-        // Apply warp prefix to our lane's partial
-        if (warp_id != 0)
-        {
-            exclusive_output = scan_op(warp_prefix, exclusive_output);
-            if (lane_id == 0)
-                exclusive_output = warp_prefix;
-        }
-    }
-
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,              ///< [in] Calling thread's input items
-        T               &exclusive_output,  ///< [out] Calling thread's output items (may be aliased to \p input)
-        const T         &initial_value,     ///< [in] Initial value to seed the exclusive scan
-        ScanOp          scan_op,            ///< [in] Binary scan operator
-        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
-    {
-        WarpScanT my_warp_scan(temp_storage.warp_scan[warp_id]);
-
-        // Compute warp scan in each warp.  The exclusive output from each lane0 is invalid.
-        T inclusive_output;
-        my_warp_scan.Scan(input, inclusive_output, exclusive_output, scan_op);
-
-        // Compute the warp-wide prefix and block-wide aggregate for each warp
-//        T warp_prefix = ComputeWarpPrefix(scan_op, inclusive_output, block_aggregate, initial_value);
-
-//--------------------------------------------------
-        // Last lane in each warp shares its warp-aggregate
-        if (lane_id == WARP_THREADS - 1)
-            temp_storage.warp_aggregates[warp_id] = inclusive_output;
-
-        CTA_SYNC();
-
-        // Get the warp scan partial
-        T warp_inclusive, warp_prefix;
-        if (lane_id < WARPS)
-        {
-            // Scan the warpscan partials
-            T warp_val = temp_storage.warp_aggregates[lane_id];
-            WarpAggregateScanT(temp_storage.inner_scan[warp_id]).Scan(warp_val, warp_inclusive, warp_prefix, initial_value, scan_op);
-        }
-
-        warp_prefix         = my_warp_scan.Broadcast(warp_prefix, warp_id);
-        block_aggregate     = my_warp_scan.Broadcast(warp_inclusive, WARPS - 1);
-//--------------------------------------------------
-
-        // Apply warp prefix to our lane's partial
-        exclusive_output = scan_op(warp_prefix, exclusive_output);
-        if (lane_id == 0)
-            exclusive_output = warp_prefix;
-    }
-
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-    template <
-        typename ScanOp,
-        typename BlockPrefixCallbackOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T                       input,                          ///< [in] Calling thread's input item
-        T                       &exclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp                  scan_op,                        ///< [in] Binary scan operator
-        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a thread block-wide prefix to be applied to all inputs.
-    {
-        // Compute block-wide exclusive scan.  The exclusive output from tid0 is invalid.
-        T block_aggregate;
-        ExclusiveScan(input, exclusive_output, scan_op, block_aggregate);
-
-        // Use the first warp to determine the thread block prefix, returning the result in lane0
-        if (warp_id == 0)
-        {
-            T block_prefix = block_prefix_callback_op(block_aggregate);
-            if (lane_id == 0)
-            {
-                // Share the prefix with all threads
-                temp_storage.block_prefix = block_prefix;
-                exclusive_output = block_prefix;                // The block prefix is the exclusive output for tid0
-            }
-        }
-
-        CTA_SYNC();
-
-        // Incorporate thread block prefix into outputs
-        T block_prefix = temp_storage.block_prefix;
-        if (linear_tid > 0)
-        {
-            exclusive_output = scan_op(block_prefix, exclusive_output);
-        }
-    }
-
-
-    //---------------------------------------------------------------------
-    // Inclusive scans
-    //---------------------------------------------------------------------
-
-    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
-    template <typename ScanOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &inclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op)                        ///< [in] Binary scan operator
-    {
-        T block_aggregate;
-        InclusiveScan(input, inclusive_output, scan_op, block_aggregate);
-    }
-
-
-    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-    template <typename ScanOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &inclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op,                        ///< [in] Binary scan operator
-        T               &block_aggregate)               ///< [out] Threadblock-wide aggregate reduction of input items
-    {
-        WarpScanT(temp_storage.warp_scan[warp_id]).InclusiveScan(input, inclusive_output, scan_op);
-
-        // Compute the warp-wide prefix and block-wide aggregate for each warp.  Warp prefix for warp0 is invalid.
-        T warp_prefix = ComputeWarpPrefix(scan_op, inclusive_output, block_aggregate);
-
-        // Apply warp prefix to our lane's partial
-        if (warp_id != 0)
-        {
-            inclusive_output = scan_op(warp_prefix, inclusive_output);
-        }
-    }
-
-
-    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-    template <
-        typename ScanOp,
-        typename BlockPrefixCallbackOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T                       input,                          ///< [in] Calling thread's input item
-        T                       &exclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp                  scan_op,                        ///< [in] Binary scan operator
-        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a thread block-wide prefix to be applied to all inputs.
-    {
-        T block_aggregate;
-        InclusiveScan(input, exclusive_output, scan_op, block_aggregate);
-
-        // Use the first warp to determine the thread block prefix, returning the result in lane0
-        if (warp_id == 0)
-        {
-            T block_prefix = block_prefix_callback_op(block_aggregate);
-            if (lane_id == 0)
-            {
-                // Share the prefix with all threads
-                temp_storage.block_prefix = block_prefix;
-            }
-        }
-
-        CTA_SYNC();
-
-        // Incorporate thread block prefix into outputs
-        T block_prefix = temp_storage.block_prefix;
-        exclusive_output = scan_op(block_prefix, exclusive_output);
-    }
-
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/block/specializations/block_scan_warp_scans3.cuh b/pfsimulator/third_party/cub/block/specializations/block_scan_warp_scans3.cuh
deleted file mode 100644
index 147ca4c5a..000000000
--- a/pfsimulator/third_party/cub/block/specializations/block_scan_warp_scans3.cuh
+++ /dev/null
@@ -1,418 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::BlockScanWarpscans provides warpscan-based variants of parallel prefix scan across a CUDA thread block.
- */
-
-#pragma once
-
-#include "../../util_arch.cuh"
-#include "../../util_ptx.cuh"
-#include "../../warp/warp_scan.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \brief BlockScanWarpScans provides warpscan-based variants of parallel prefix scan across a CUDA thread block.
- */
-template <
-    typename    T,
-    int         BLOCK_DIM_X,    ///< The thread block length in threads along the X dimension
-    int         BLOCK_DIM_Y,    ///< The thread block length in threads along the Y dimension
-    int         BLOCK_DIM_Z,    ///< The thread block length in threads along the Z dimension
-    int         PTX_ARCH>       ///< The PTX compute capability for which to to specialize this collective
-struct BlockScanWarpScans
-{
-    //---------------------------------------------------------------------
-    // Types and constants
-    //---------------------------------------------------------------------
-
-    /// Constants
-    enum
-    {
-        /// The thread block size in threads
-        BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
-
-        /// Number of warp threads
-        INNER_WARP_THREADS = CUB_WARP_THREADS(PTX_ARCH),
-        OUTER_WARP_THREADS = BLOCK_THREADS / INNER_WARP_THREADS,
-
-        /// Number of outer scan warps
-        OUTER_WARPS = INNER_WARP_THREADS
-    };
-
-    ///  Outer WarpScan utility type
-    typedef WarpScan<T, OUTER_WARP_THREADS, PTX_ARCH> OuterWarpScanT;
-
-    ///  Inner WarpScan utility type
-    typedef WarpScan<T, INNER_WARP_THREADS, PTX_ARCH> InnerWarpScanT;
-
-    typedef typename OuterWarpScanT::TempStorage OuterScanArray[OUTER_WARPS];
-
-
-    /// Shared memory storage layout type
-    struct _TempStorage
-    {
-        union Aliasable
-        {
-            Uninitialized<OuterScanArray>           outer_warp_scan;  ///< Buffer for warp-synchronous outer scans
-            typename InnerWarpScanT::TempStorage    inner_warp_scan;  ///< Buffer for warp-synchronous inner scan
-
-        } aliasable;
-
-        T                               warp_aggregates[OUTER_WARPS];
-
-        T                               block_aggregate;                           ///< Shared prefix for the entire thread block
-    };
-
-
-    /// Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    //---------------------------------------------------------------------
-    // Per-thread fields
-    //---------------------------------------------------------------------
-
-    // Thread fields
-    _TempStorage    &temp_storage;
-    unsigned int    linear_tid;
-    unsigned int    warp_id;
-    unsigned int    lane_id;
-
-
-    //---------------------------------------------------------------------
-    // Constructors
-    //---------------------------------------------------------------------
-
-    /// Constructor
-    __device__ __forceinline__ BlockScanWarpScans(
-        TempStorage &temp_storage)
-    :
-        temp_storage(temp_storage.Alias()),
-        linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z)),
-        warp_id((OUTER_WARPS == 1) ? 0 : linear_tid / OUTER_WARP_THREADS),
-        lane_id((OUTER_WARPS == 1) ? linear_tid : linear_tid % OUTER_WARP_THREADS)
-    {}
-
-
-    //---------------------------------------------------------------------
-    // Exclusive scans
-    //---------------------------------------------------------------------
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &exclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op)                        ///< [in] Binary scan operator
-    {
-        // Compute block-wide exclusive scan.  The exclusive output from tid0 is invalid.
-        T block_aggregate;
-        ExclusiveScan(input, exclusive_output, scan_op, block_aggregate);
-    }
-
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,              ///< [in] Calling thread's input items
-        T               &exclusive_output,  ///< [out] Calling thread's output items (may be aliased to \p input)
-        const T         &initial_value,     ///< [in] Initial value to seed the exclusive scan
-        ScanOp          scan_op)            ///< [in] Binary scan operator
-    {
-        T block_aggregate;
-        ExclusiveScan(input, exclusive_output, initial_value, scan_op, block_aggregate);
-    }
-
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.  With no initial value, the output computed for <em>thread</em><sub>0</sub> is undefined.
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,              ///< [in] Calling thread's input item
-        T               &exclusive_output,  ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op,            ///< [in] Binary scan operator
-        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
-    {
-        // Compute warp scan in each warp.  The exclusive output from each lane0 is invalid.
-        T inclusive_output;
-        OuterWarpScanT(temp_storage.aliasable.outer_warp_scan.Alias()[warp_id]).Scan(
-            input, inclusive_output, exclusive_output, scan_op);
-
-        // Share outer warp total
-        if (lane_id == OUTER_WARP_THREADS - 1)
-            temp_storage.warp_aggregates[warp_id] = inclusive_output;
-
-        CTA_SYNC();
-
-        if (linear_tid < INNER_WARP_THREADS)
-        {
-            T outer_warp_input = temp_storage.warp_aggregates[linear_tid];
-            T outer_warp_exclusive;
-
-            InnerWarpScanT(temp_storage.aliasable.inner_warp_scan).ExclusiveScan(
-                outer_warp_input, outer_warp_exclusive, scan_op, block_aggregate);
-
-            temp_storage.block_aggregate                = block_aggregate;
-            temp_storage.warp_aggregates[linear_tid]    = outer_warp_exclusive;
-        }
-
-        CTA_SYNC();
-
-        if (warp_id != 0)
-        {
-            // Retrieve block aggregate
-            block_aggregate = temp_storage.block_aggregate;
-
-            // Apply warp prefix to our lane's partial
-            T outer_warp_exclusive = temp_storage.warp_aggregates[warp_id];
-            exclusive_output = scan_op(outer_warp_exclusive, exclusive_output);
-            if (lane_id == 0)
-                exclusive_output = outer_warp_exclusive;
-        }
-    }
-
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,              ///< [in] Calling thread's input items
-        T               &exclusive_output,  ///< [out] Calling thread's output items (may be aliased to \p input)
-        const T         &initial_value,     ///< [in] Initial value to seed the exclusive scan
-        ScanOp          scan_op,            ///< [in] Binary scan operator
-        T               &block_aggregate)   ///< [out] Threadblock-wide aggregate reduction of input items
-    {
-        // Compute warp scan in each warp.  The exclusive output from each lane0 is invalid.
-        T inclusive_output;
-        OuterWarpScanT(temp_storage.aliasable.outer_warp_scan.Alias()[warp_id]).Scan(
-            input, inclusive_output, exclusive_output, scan_op);
-
-        // Share outer warp total
-        if (lane_id == OUTER_WARP_THREADS - 1)
-        {
-            temp_storage.warp_aggregates[warp_id] = inclusive_output;
-        }
-
-        CTA_SYNC();
-
-        if (linear_tid < INNER_WARP_THREADS)
-        {
-            T outer_warp_input = temp_storage.warp_aggregates[linear_tid];
-            T outer_warp_exclusive;
-
-            InnerWarpScanT(temp_storage.aliasable.inner_warp_scan).ExclusiveScan(
-                outer_warp_input, outer_warp_exclusive, initial_value, scan_op, block_aggregate);
-
-            temp_storage.block_aggregate                = block_aggregate;
-            temp_storage.warp_aggregates[linear_tid]    = outer_warp_exclusive;
-        }
-
-        CTA_SYNC();
-
-        // Retrieve block aggregate
-        block_aggregate = temp_storage.block_aggregate;
-
-        // Apply warp prefix to our lane's partial
-        T outer_warp_exclusive = temp_storage.warp_aggregates[warp_id];
-        exclusive_output = scan_op(outer_warp_exclusive, exclusive_output);
-        if (lane_id == 0)
-            exclusive_output = outer_warp_exclusive;
-    }
-
-
-    /// Computes an exclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  The call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.
-    template <
-        typename ScanOp,
-        typename BlockPrefixCallbackOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T                       input,                          ///< [in] Calling thread's input item
-        T                       &exclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp                  scan_op,                        ///< [in] Binary scan operator
-        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a thread block-wide prefix to be applied to all inputs.
-    {
-        // Compute warp scan in each warp.  The exclusive output from each lane0 is invalid.
-        T inclusive_output;
-        OuterWarpScanT(temp_storage.aliasable.outer_warp_scan.Alias()[warp_id]).Scan(
-            input, inclusive_output, exclusive_output, scan_op);
-
-        // Share outer warp total
-        if (lane_id == OUTER_WARP_THREADS - 1)
-            temp_storage.warp_aggregates[warp_id] = inclusive_output;
-
-        CTA_SYNC();
-
-        if (linear_tid < INNER_WARP_THREADS)
-        {
-            InnerWarpScanT inner_scan(temp_storage.aliasable.inner_warp_scan);
-
-            T upsweep = temp_storage.warp_aggregates[linear_tid];
-            T downsweep_prefix, block_aggregate;
-
-            inner_scan.ExclusiveScan(upsweep, downsweep_prefix, scan_op, block_aggregate);
-
-            // Use callback functor to get block prefix in lane0 and then broadcast to other lanes
-            T block_prefix = block_prefix_callback_op(block_aggregate);
-            block_prefix = inner_scan.Broadcast(block_prefix, 0);
-
-            downsweep_prefix = scan_op(block_prefix, downsweep_prefix);
-            if (linear_tid == 0)
-                downsweep_prefix = block_prefix;
-
-            temp_storage.warp_aggregates[linear_tid] = downsweep_prefix;
-        }
-
-        CTA_SYNC();
-
-        // Apply warp prefix to our lane's partial (or assign it if partial is invalid)
-        T outer_warp_exclusive = temp_storage.warp_aggregates[warp_id];
-        exclusive_output = scan_op(outer_warp_exclusive, exclusive_output);
-        if (lane_id == 0)
-            exclusive_output = outer_warp_exclusive;
-    }
-
-
-    //---------------------------------------------------------------------
-    // Inclusive scans
-    //---------------------------------------------------------------------
-
-    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.
-    template <typename ScanOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &inclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op)                        ///< [in] Binary scan operator
-    {
-        T block_aggregate;
-        InclusiveScan(input, inclusive_output, scan_op, block_aggregate);
-    }
-
-
-    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  Also provides every thread with the block-wide \p block_aggregate of all inputs.
-    template <typename ScanOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T               input,                          ///< [in] Calling thread's input item
-        T               &inclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp          scan_op,                        ///< [in] Binary scan operator
-        T               &block_aggregate)               ///< [out] Threadblock-wide aggregate reduction of input items
-    {
-        // Compute warp scan in each warp.  The exclusive output from each lane0 is invalid.
-        OuterWarpScanT(temp_storage.aliasable.outer_warp_scan.Alias()[warp_id]).InclusiveScan(
-            input, inclusive_output, scan_op);
-
-        // Share outer warp total
-        if (lane_id == OUTER_WARP_THREADS - 1)
-            temp_storage.warp_aggregates[warp_id] = inclusive_output;
-
-        CTA_SYNC();
-
-        if (linear_tid < INNER_WARP_THREADS)
-        {
-            T outer_warp_input = temp_storage.warp_aggregates[linear_tid];
-            T outer_warp_exclusive;
-
-            InnerWarpScanT(temp_storage.aliasable.inner_warp_scan).ExclusiveScan(
-                outer_warp_input, outer_warp_exclusive, scan_op, block_aggregate);
-
-            temp_storage.block_aggregate                = block_aggregate;
-            temp_storage.warp_aggregates[linear_tid]    = outer_warp_exclusive;
-        }
-
-        CTA_SYNC();
-
-        if (warp_id != 0)
-        {
-            // Retrieve block aggregate
-            block_aggregate = temp_storage.block_aggregate;
-
-            // Apply warp prefix to our lane's partial
-            T outer_warp_exclusive = temp_storage.warp_aggregates[warp_id];
-            inclusive_output = scan_op(outer_warp_exclusive, inclusive_output);
-        }
-    }
-
-
-    /// Computes an inclusive thread block-wide prefix scan using the specified binary \p scan_op functor.  Each thread contributes one input element.  the call-back functor \p block_prefix_callback_op is invoked by the first warp in the block, and the value returned by <em>lane</em><sub>0</sub> in that warp is used as the "seed" value that logically prefixes the thread block's scan inputs.
-    template <
-        typename ScanOp,
-        typename BlockPrefixCallbackOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T                       input,                          ///< [in] Calling thread's input item
-        T                       &inclusive_output,              ///< [out] Calling thread's output item (may be aliased to \p input)
-        ScanOp                  scan_op,                        ///< [in] Binary scan operator
-        BlockPrefixCallbackOp   &block_prefix_callback_op)      ///< [in-out] <b>[<em>warp</em><sub>0</sub> only]</b> Call-back functor for specifying a thread block-wide prefix to be applied to all inputs.
-    {
-        // Compute warp scan in each warp.  The exclusive output from each lane0 is invalid.
-        OuterWarpScanT(temp_storage.aliasable.outer_warp_scan.Alias()[warp_id]).InclusiveScan(
-            input, inclusive_output, scan_op);
-
-        // Share outer warp total
-        if (lane_id == OUTER_WARP_THREADS - 1)
-            temp_storage.warp_aggregates[warp_id] = inclusive_output;
-
-        CTA_SYNC();
-
-        if (linear_tid < INNER_WARP_THREADS)
-        {
-            InnerWarpScanT inner_scan(temp_storage.aliasable.inner_warp_scan);
-
-            T upsweep = temp_storage.warp_aggregates[linear_tid];
-            T downsweep_prefix, block_aggregate;
-            inner_scan.ExclusiveScan(upsweep, downsweep_prefix, scan_op, block_aggregate);
-
-            // Use callback functor to get block prefix in lane0 and then broadcast to other lanes
-            T block_prefix = block_prefix_callback_op(block_aggregate);
-            block_prefix = inner_scan.Broadcast(block_prefix, 0);
-
-            downsweep_prefix = scan_op(block_prefix, downsweep_prefix);
-            if (linear_tid == 0)
-                downsweep_prefix = block_prefix;
-
-            temp_storage.warp_aggregates[linear_tid]    = downsweep_prefix;
-        }
-
-        CTA_SYNC();
-
-        // Apply warp prefix to our lane's partial
-        T outer_warp_exclusive = temp_storage.warp_aggregates[warp_id];
-        inclusive_output = scan_op(outer_warp_exclusive, inclusive_output);
-    }
-
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/cub.cuh b/pfsimulator/third_party/cub/cub.cuh
deleted file mode 100644
index 3ece0f658..000000000
--- a/pfsimulator/third_party/cub/cub.cuh
+++ /dev/null
@@ -1,95 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * CUB umbrella include file
- */
-
-#pragma once
-
-
-// Block
-#include "block/block_histogram.cuh"
-#include "block/block_discontinuity.cuh"
-#include "block/block_exchange.cuh"
-#include "block/block_load.cuh"
-#include "block/block_radix_rank.cuh"
-#include "block/block_radix_sort.cuh"
-#include "block/block_reduce.cuh"
-#include "block/block_scan.cuh"
-#include "block/block_store.cuh"
-//#include "block/block_shift.cuh"
-
-// Device
-#include "device/device_histogram.cuh"
-#include "device/device_partition.cuh"
-#include "device/device_radix_sort.cuh"
-#include "device/device_reduce.cuh"
-#include "device/device_run_length_encode.cuh"
-#include "device/device_scan.cuh"
-#include "device/device_segmented_radix_sort.cuh"
-#include "device/device_segmented_reduce.cuh"
-#include "device/device_select.cuh"
-#include "device/device_spmv.cuh"
-
-// Grid
-//#include "grid/grid_barrier.cuh"
-#include "grid/grid_even_share.cuh"
-#include "grid/grid_mapping.cuh"
-#include "grid/grid_queue.cuh"
-
-// Thread
-#include "thread/thread_load.cuh"
-#include "thread/thread_operators.cuh"
-#include "thread/thread_reduce.cuh"
-#include "thread/thread_scan.cuh"
-#include "thread/thread_store.cuh"
-
-// Warp
-#include "warp/warp_reduce.cuh"
-#include "warp/warp_scan.cuh"
-
-// Iterator
-#include "iterator/arg_index_input_iterator.cuh"
-#include "iterator/cache_modified_input_iterator.cuh"
-#include "iterator/cache_modified_output_iterator.cuh"
-#include "iterator/constant_input_iterator.cuh"
-#include "iterator/counting_input_iterator.cuh"
-#include "iterator/tex_obj_input_iterator.cuh"
-#include "iterator/tex_ref_input_iterator.cuh"
-#include "iterator/transform_input_iterator.cuh"
-
-// Util
-#include "util_arch.cuh"
-#include "util_debug.cuh"
-#include "util_device.cuh"
-#include "util_macro.cuh"
-#include "util_ptx.cuh"
-#include "util_type.cuh"
-
diff --git a/pfsimulator/third_party/cub/device/device_histogram.cuh b/pfsimulator/third_party/cub/device/device_histogram.cuh
deleted file mode 100644
index a2556a6b8..000000000
--- a/pfsimulator/third_party/cub/device/device_histogram.cuh
+++ /dev/null
@@ -1,866 +0,0 @@
-
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::DeviceHistogram provides device-wide parallel operations for constructing histogram(s) from a sequence of samples data residing within device-accessible memory.
- */
-
-#pragma once
-
-#include <stdio.h>
-#include <iterator>
-#include <limits>
-
-#include "dispatch/dispatch_histogram.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \brief DeviceHistogram provides device-wide parallel operations for constructing histogram(s) from a sequence of samples data residing within device-accessible memory. ![](histogram_logo.png)
- * \ingroup SingleModule
- *
- * \par Overview
- * A <a href="http://en.wikipedia.org/wiki/Histogram"><em>histogram</em></a>
- * counts the number of observations that fall into each of the disjoint categories (known as <em>bins</em>).
- *
- * \par Usage Considerations
- * \cdp_class{DeviceHistogram}
- *
- */
-struct DeviceHistogram
-{
-    /******************************************************************//**
-     * \name Evenly-segmented bin ranges
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Computes an intensity histogram from a sequence of data samples using equal-width bins.
-     *
-     * \par
-     * - The number of histogram bins is (\p num_levels - 1)
-     * - All bins comprise the same width of sample values: (\p upper_level - \p lower_level) / (\p num_levels - 1)
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the computation of a six-bin histogram
-     * from a sequence of float samples
-     *
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_histogram.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input samples and
-     * // output histogram
-     * int      num_samples;    // e.g., 10
-     * float*   d_samples;      // e.g., [2.2, 6.0, 7.1, 2.9, 3.5, 0.3, 2.9, 2.0, 6.1, 999.5]
-     * int*     d_histogram;    // e.g., [ -, -, -, -, -, -, -, -]
-     * int      num_levels;     // e.g., 7       (seven level boundaries for six bins)
-     * float    lower_level;    // e.g., 0.0     (lower sample value boundary of lowest bin)
-     * float    upper_level;    // e.g., 12.0    (upper sample value boundary of upper bin)
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void*    d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceHistogram::HistogramEven(d_temp_storage, temp_storage_bytes,
-     *     d_samples, d_histogram, num_levels, lower_level, upper_level, num_samples);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Compute histograms
-     * cub::DeviceHistogram::HistogramEven(d_temp_storage, temp_storage_bytes,
-     *     d_samples, d_histogram, num_levels, lower_level, upper_level, num_samples);
-     *
-     * // d_histogram   <-- [1, 0, 5, 0, 3, 0, 0, 0];
-     *
-     * \endcode
-     *
-     * \tparam SampleIteratorT          <b>[inferred]</b> Random-access input iterator type for reading input samples. \iterator
-     * \tparam CounterT                 <b>[inferred]</b> Integer type for histogram bin counters
-     * \tparam LevelT                   <b>[inferred]</b> Type for specifying boundaries (levels)
-     * \tparam OffsetT                  <b>[inferred]</b> Signed integer type for sequence offsets, list lengths, pointer differences, etc.  \offset_size1
-     */
-    template <
-        typename            SampleIteratorT,
-        typename            CounterT,
-        typename            LevelT,
-        typename            OffsetT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t HistogramEven(
-        void*               d_temp_storage,                             ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&             temp_storage_bytes,                        ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        SampleIteratorT     d_samples,                                  ///< [in] The pointer to the input sequence of data samples.
-        CounterT*           d_histogram,                                ///< [out] The pointer to the histogram counter output array of length <tt>num_levels</tt> - 1.
-        int                 num_levels,                                 ///< [in] The number of boundaries (levels) for delineating histogram samples.  Implies that the number of bins is <tt>num_levels</tt> - 1.
-        LevelT              lower_level,                                ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin.
-        LevelT              upper_level,                                ///< [in] The upper sample value bound (exclusive) for the highest histogram bin.
-        OffsetT             num_samples,                                ///< [in] The number of input samples (i.e., the length of \p d_samples)
-        cudaStream_t        stream                  = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous       = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        /// The sample value type of the input iterator
-        typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
-
-        CounterT*           d_histogram1[1]     = {d_histogram};
-        int                 num_levels1[1]      = {num_levels};
-        LevelT              lower_level1[1]     = {lower_level};
-        LevelT              upper_level1[1]     = {upper_level};
-
-        return MultiHistogramEven<1, 1>(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_samples,
-            d_histogram1,
-            num_levels1,
-            lower_level1,
-            upper_level1,
-            num_samples,
-            1,
-            sizeof(SampleT) * num_samples,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Computes an intensity histogram from a sequence of data samples using equal-width bins.
-     *
-     * \par
-     * - A two-dimensional <em>region of interest</em> within \p d_samples can be specified
-     *   using the \p num_row_samples, num_rows, and \p row_stride_bytes parameters.
-     * - The row stride must be a whole multiple of the sample data type
-     *   size, i.e., <tt>(row_stride_bytes % sizeof(SampleT)) == 0</tt>.
-     * - The number of histogram bins is (\p num_levels - 1)
-     * - All bins comprise the same width of sample values: (\p upper_level - \p lower_level) / (\p num_levels - 1)
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the computation of a six-bin histogram
-     * from a 2x5 region of interest within a flattened 2x7 array of float samples.
-     *
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_histogram.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input samples and
-     * // output histogram
-     * int      num_row_samples;    // e.g., 5
-     * int      num_rows;           // e.g., 2;
-     * size_t   row_stride_bytes;   // e.g., 7 * sizeof(float)
-     * float*   d_samples;          // e.g., [2.2, 6.0, 7.1, 2.9, 3.5,   -, -,
-     *                              //        0.3, 2.9, 2.0, 6.1, 999.5, -, -]
-     * int*     d_histogram;        // e.g., [ -, -, -, -, -, -, -, -]
-     * int      num_levels;         // e.g., 7       (seven level boundaries for six bins)
-     * float    lower_level;        // e.g., 0.0     (lower sample value boundary of lowest bin)
-     * float    upper_level;        // e.g., 12.0    (upper sample value boundary of upper bin)
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void*    d_temp_storage  = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceHistogram::HistogramEven(d_temp_storage, temp_storage_bytes,
-     *     d_samples, d_histogram, num_levels, lower_level, upper_level,
-     *     num_row_samples, num_rows, row_stride_bytes);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Compute histograms
-     * cub::DeviceHistogram::HistogramEven(d_temp_storage, temp_storage_bytes, d_samples, d_histogram,
-     *     d_samples, d_histogram, num_levels, lower_level, upper_level,
-     *     num_row_samples, num_rows, row_stride_bytes);
-     *
-     * // d_histogram   <-- [1, 0, 5, 0, 3, 0, 0, 0];
-     *
-     * \endcode
-     *
-     * \tparam SampleIteratorT          <b>[inferred]</b> Random-access input iterator type for reading input samples. \iterator
-     * \tparam CounterT                 <b>[inferred]</b> Integer type for histogram bin counters
-     * \tparam LevelT                   <b>[inferred]</b> Type for specifying boundaries (levels)
-     * \tparam OffsetT                  <b>[inferred]</b> Signed integer type for sequence offsets, list lengths, pointer differences, etc.  \offset_size1
-     */
-    template <
-        typename            SampleIteratorT,
-        typename            CounterT,
-        typename            LevelT,
-        typename            OffsetT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t HistogramEven(
-        void*               d_temp_storage,                             ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&             temp_storage_bytes,                        ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        SampleIteratorT     d_samples,                                  ///< [in] The pointer to the input sequence of data samples.
-        CounterT*           d_histogram,                                ///< [out] The pointer to the histogram counter output array of length <tt>num_levels</tt> - 1.
-        int                 num_levels,                                 ///< [in] The number of boundaries (levels) for delineating histogram samples.  Implies that the number of bins is <tt>num_levels</tt> - 1.
-        LevelT              lower_level,                                ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin.
-        LevelT              upper_level,                                ///< [in] The upper sample value bound (exclusive) for the highest histogram bin.
-        OffsetT             num_row_samples,                            ///< [in] The number of data samples per row in the region of interest
-        OffsetT             num_rows,                                   ///< [in] The number of rows in the region of interest
-        size_t              row_stride_bytes,                           ///< [in] The number of bytes between starts of consecutive rows in the region of interest
-        cudaStream_t        stream                  = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous       = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        CounterT*           d_histogram1[1]     = {d_histogram};
-        int                 num_levels1[1]      = {num_levels};
-        LevelT              lower_level1[1]     = {lower_level};
-        LevelT              upper_level1[1]     = {upper_level};
-
-        return MultiHistogramEven<1, 1>(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_samples,
-            d_histogram1,
-            num_levels1,
-            lower_level1,
-            upper_level1,
-            num_row_samples,
-            num_rows,
-            row_stride_bytes,
-            stream,
-            debug_synchronous);
-    }
-
-    /**
-     * \brief Computes per-channel intensity histograms from a sequence of multi-channel "pixel" data samples using equal-width bins.
-     *
-     * \par
-     * - The input is a sequence of <em>pixel</em> structures, where each pixel comprises
-     *   a record of \p NUM_CHANNELS consecutive data samples (e.g., an <em>RGBA</em> pixel).
-     * - Of the \p NUM_CHANNELS specified, the function will only compute histograms
-     *   for the first \p NUM_ACTIVE_CHANNELS (e.g., only <em>RGB</em> histograms from <em>RGBA</em>
-     *   pixel samples).
-     * - The number of histogram bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
-     * - For channel<sub><em>i</em></sub>, the range of values for all histogram bins
-     *   have the same width: (<tt>upper_level[i]</tt> - <tt>lower_level[i]</tt>) / (<tt> num_levels[i]</tt> - 1)
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the computation of three 256-bin <em>RGB</em> histograms
-     * from a quad-channel sequence of <em>RGBA</em> pixels (8 bits per channel per pixel)
-     *
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_histogram.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input samples
-     * // and output histograms
-     * int              num_pixels;         // e.g., 5
-     * unsigned char*   d_samples;          // e.g., [(2, 6, 7, 5), (3, 0, 2, 1), (7, 0, 6, 2),
-     *                                      //        (0, 6, 7, 5), (3, 0, 2, 6)]
-     * int*             d_histogram[3];     // e.g., three device pointers to three device buffers,
-     *                                      //       each allocated with 256 integer counters
-     * int              num_levels[3];      // e.g., {257, 257, 257};
-     * unsigned int     lower_level[3];     // e.g., {0, 0, 0};
-     * unsigned int     upper_level[3];     // e.g., {256, 256, 256};
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void*    d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceHistogram::MultiHistogramEven<4, 3>(d_temp_storage, temp_storage_bytes,
-     *     d_samples, d_histogram, num_levels, lower_level, upper_level, num_pixels);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Compute histograms
-     * cub::DeviceHistogram::MultiHistogramEven<4, 3>(d_temp_storage, temp_storage_bytes,
-     *     d_samples, d_histogram, num_levels, lower_level, upper_level, num_pixels);
-     *
-     * // d_histogram   <-- [ [1, 0, 1, 2, 0, 0, 0, 1, 0, 0, 0, ..., 0],
-     * //                     [0, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, ..., 0],
-     * //                     [0, 0, 2, 0, 0, 0, 1, 2, 0, 0, 0, ..., 0] ]
-     *
-     * \endcode
-     *
-     * \tparam NUM_CHANNELS             Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed)
-     * \tparam NUM_ACTIVE_CHANNELS      <b>[inferred]</b> Number of channels actively being histogrammed
-     * \tparam SampleIteratorT          <b>[inferred]</b> Random-access input iterator type for reading input samples. \iterator
-     * \tparam CounterT                 <b>[inferred]</b> Integer type for histogram bin counters
-     * \tparam LevelT                   <b>[inferred]</b> Type for specifying boundaries (levels)
-     * \tparam OffsetT                  <b>[inferred]</b> Signed integer type for sequence offsets, list lengths, pointer differences, etc.  \offset_size1
-     */
-    template <
-        int                 NUM_CHANNELS,
-        int                 NUM_ACTIVE_CHANNELS,
-        typename            SampleIteratorT,
-        typename            CounterT,
-        typename            LevelT,
-        typename            OffsetT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t MultiHistogramEven(
-        void*               d_temp_storage,                             ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&             temp_storage_bytes,                        ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        SampleIteratorT     d_samples,                                  ///< [in] The pointer to the multi-channel input sequence of data samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four <em>RGBA</em> 8-bit samples).
-        CounterT*           d_histogram[NUM_ACTIVE_CHANNELS],           ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histogram[i]</tt> should be <tt>num_levels[i]</tt> - 1.
-        int                 num_levels[NUM_ACTIVE_CHANNELS],            ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
-        LevelT              lower_level[NUM_ACTIVE_CHANNELS],           ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin in each active channel.
-        LevelT              upper_level[NUM_ACTIVE_CHANNELS],           ///< [in] The upper sample value bound (exclusive) for the highest histogram bin in each active channel.
-        OffsetT             num_pixels,                                 ///< [in] The number of multi-channel pixels (i.e., the length of \p d_samples / NUM_CHANNELS)
-        cudaStream_t        stream                  = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous       = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        /// The sample value type of the input iterator
-        typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
-
-        return MultiHistogramEven<NUM_CHANNELS, NUM_ACTIVE_CHANNELS>(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_samples,
-            d_histogram,
-            num_levels,
-            lower_level,
-            upper_level,
-            num_pixels,
-            1,
-            sizeof(SampleT) * NUM_CHANNELS * num_pixels,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Computes per-channel intensity histograms from a sequence of multi-channel "pixel" data samples using equal-width bins.
-     *
-     * \par
-     * - The input is a sequence of <em>pixel</em> structures, where each pixel comprises
-     *   a record of \p NUM_CHANNELS consecutive data samples (e.g., an <em>RGBA</em> pixel).
-     * - Of the \p NUM_CHANNELS specified, the function will only compute histograms
-     *   for the first \p NUM_ACTIVE_CHANNELS (e.g., only <em>RGB</em> histograms from <em>RGBA</em>
-     *   pixel samples).
-     * - A two-dimensional <em>region of interest</em> within \p d_samples can be specified
-     *   using the \p num_row_samples, num_rows, and \p row_stride_bytes parameters.
-     * - The row stride must be a whole multiple of the sample data type
-     *   size, i.e., <tt>(row_stride_bytes % sizeof(SampleT)) == 0</tt>.
-     * - The number of histogram bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
-     * - For channel<sub><em>i</em></sub>, the range of values for all histogram bins
-     *   have the same width: (<tt>upper_level[i]</tt> - <tt>lower_level[i]</tt>) / (<tt> num_levels[i]</tt> - 1)
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the computation of three 256-bin <em>RGB</em> histograms from a 2x3 region of
-     * interest of within a flattened 2x4 array of quad-channel <em>RGBA</em> pixels (8 bits per channel per pixel).
-     *
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_histogram.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input samples
-     * // and output histograms
-     * int              num_row_pixels;     // e.g., 3
-     * int              num_rows;           // e.g., 2
-     * size_t           row_stride_bytes;   // e.g., 4 * sizeof(unsigned char) * NUM_CHANNELS
-     * unsigned char*   d_samples;          // e.g., [(2, 6, 7, 5), (3, 0, 2, 1), (7, 0, 6, 2), (-, -, -, -),
-     *                                      //        (0, 6, 7, 5), (3, 0, 2, 6), (1, 1, 1, 1), (-, -, -, -)]
-     * int*             d_histogram[3];     // e.g., three device pointers to three device buffers,
-     *                                      //       each allocated with 256 integer counters
-     * int              num_levels[3];      // e.g., {257, 257, 257};
-     * unsigned int     lower_level[3];     // e.g., {0, 0, 0};
-     * unsigned int     upper_level[3];     // e.g., {256, 256, 256};
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void*    d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceHistogram::MultiHistogramEven<4, 3>(d_temp_storage, temp_storage_bytes,
-     *     d_samples, d_histogram, num_levels, lower_level, upper_level,
-     *     num_row_pixels, num_rows, row_stride_bytes);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Compute histograms
-     * cub::DeviceHistogram::MultiHistogramEven<4, 3>(d_temp_storage, temp_storage_bytes,
-     *     d_samples, d_histogram, num_levels, lower_level, upper_level,
-     *     num_row_pixels, num_rows, row_stride_bytes);
-     *
-     * // d_histogram   <-- [ [1, 1, 1, 2, 0, 0, 0, 1, 0, 0, 0, ..., 0],
-     * //                     [0, 4, 0, 0, 0, 0, 2, 0, 0, 0, 0, ..., 0],
-     * //                     [0, 1, 2, 0, 0, 0, 1, 2, 0, 0, 0, ..., 0] ]
-     *
-     * \endcode
-     *
-     * \tparam NUM_CHANNELS             Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed)
-     * \tparam NUM_ACTIVE_CHANNELS      <b>[inferred]</b> Number of channels actively being histogrammed
-     * \tparam SampleIteratorT          <b>[inferred]</b> Random-access input iterator type for reading input samples. \iterator
-     * \tparam CounterT                 <b>[inferred]</b> Integer type for histogram bin counters
-     * \tparam LevelT                   <b>[inferred]</b> Type for specifying boundaries (levels)
-     * \tparam OffsetT                  <b>[inferred]</b> Signed integer type for sequence offsets, list lengths, pointer differences, etc.  \offset_size1
-     */
-    template <
-        int                 NUM_CHANNELS,
-        int                 NUM_ACTIVE_CHANNELS,
-        typename            SampleIteratorT,
-        typename            CounterT,
-        typename            LevelT,
-        typename            OffsetT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t MultiHistogramEven(
-        void*               d_temp_storage,                             ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&             temp_storage_bytes,                        ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        SampleIteratorT     d_samples,                                  ///< [in] The pointer to the multi-channel input sequence of data samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four <em>RGBA</em> 8-bit samples).
-        CounterT*           d_histogram[NUM_ACTIVE_CHANNELS],           ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histogram[i]</tt> should be <tt>num_levels[i]</tt> - 1.
-        int                 num_levels[NUM_ACTIVE_CHANNELS],            ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
-        LevelT              lower_level[NUM_ACTIVE_CHANNELS],           ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin in each active channel.
-        LevelT              upper_level[NUM_ACTIVE_CHANNELS],           ///< [in] The upper sample value bound (exclusive) for the highest histogram bin in each active channel.
-        OffsetT             num_row_pixels,                             ///< [in] The number of multi-channel pixels per row in the region of interest
-        OffsetT             num_rows,                                   ///< [in] The number of rows in the region of interest
-        size_t              row_stride_bytes,                           ///< [in] The number of bytes between starts of consecutive rows in the region of interest
-        cudaStream_t        stream                  = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous       = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        /// The sample value type of the input iterator
-        typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
-        Int2Type<sizeof(SampleT) == 1> is_byte_sample;
-
-        if ((sizeof(OffsetT) > sizeof(int)) &&
-            ((unsigned long long) (num_rows * row_stride_bytes) < (unsigned long long) std::numeric_limits<int>::max()))
-        {
-            // Down-convert OffsetT data type
-
-
-            return DipatchHistogram<NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, LevelT, int>::DispatchEven(
-                d_temp_storage, temp_storage_bytes, d_samples, d_histogram, num_levels, lower_level, upper_level,
-                (int) num_row_pixels, (int) num_rows, (int) (row_stride_bytes / sizeof(SampleT)),
-                stream, debug_synchronous, is_byte_sample);
-        }
-
-        return DipatchHistogram<NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, LevelT, OffsetT>::DispatchEven(
-            d_temp_storage, temp_storage_bytes, d_samples, d_histogram, num_levels, lower_level, upper_level,
-            num_row_pixels, num_rows, (OffsetT) (row_stride_bytes / sizeof(SampleT)),
-            stream, debug_synchronous, is_byte_sample);
-    }
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Custom bin ranges
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Computes an intensity histogram from a sequence of data samples using the specified bin boundary levels.
-     *
-     * \par
-     * - The number of histogram bins is (\p num_levels - 1)
-     * - The value range for bin<sub><em>i</em></sub> is [<tt>level[i]</tt>, <tt>level[i+1]</tt>)
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the computation of an six-bin histogram
-     * from a sequence of float samples
-     *
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_histogram.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input samples and
-     * // output histogram
-     * int      num_samples;    // e.g., 10
-     * float*   d_samples;      // e.g., [2.2, 6.0, 7.1, 2.9, 3.5, 0.3, 2.9, 2.0, 6.1, 999.5]
-     * int*     d_histogram;    // e.g., [ -, -, -, -, -, -, -, -]
-     * int      num_levels      // e.g., 7 (seven level boundaries for six bins)
-     * float*   d_levels;       // e.g., [0.0, 2.0, 4.0, 6.0, 8.0, 12.0, 16.0]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void*    d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceHistogram::HistogramRange(d_temp_storage, temp_storage_bytes,
-     *     d_samples, d_histogram, num_levels, d_levels, num_samples);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Compute histograms
-     * cub::DeviceHistogram::HistogramRange(d_temp_storage, temp_storage_bytes,
-     *     d_samples, d_histogram, num_levels, d_levels, num_samples);
-     *
-     * // d_histogram   <-- [1, 0, 5, 0, 3, 0, 0, 0];
-     *
-     * \endcode
-     *
-     * \tparam SampleIteratorT          <b>[inferred]</b> Random-access input iterator type for reading input samples. \iterator
-     * \tparam CounterT                 <b>[inferred]</b> Integer type for histogram bin counters
-     * \tparam LevelT                   <b>[inferred]</b> Type for specifying boundaries (levels)
-     * \tparam OffsetT                  <b>[inferred]</b> Signed integer type for sequence offsets, list lengths, pointer differences, etc.  \offset_size1
-     */
-    template <
-        typename            SampleIteratorT,
-        typename            CounterT,
-        typename            LevelT,
-        typename            OffsetT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t HistogramRange(
-        void*               d_temp_storage,                         ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&             temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        SampleIteratorT     d_samples,                              ///< [in] The pointer to the input sequence of data samples.
-        CounterT*           d_histogram,                            ///< [out] The pointer to the histogram counter output array of length <tt>num_levels</tt> - 1.
-        int                 num_levels,                             ///< [in] The number of boundaries (levels) for delineating histogram samples.  Implies that the number of bins is <tt>num_levels</tt> - 1.
-        LevelT*             d_levels,                               ///< [in] The pointer to the array of boundaries (levels).  Bin ranges are defined by consecutive boundary pairings: lower sample value boundaries are inclusive and upper sample value boundaries are exclusive.
-        OffsetT             num_samples,                            ///< [in] The number of data samples per row in the region of interest
-        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        /// The sample value type of the input iterator
-        typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
-
-        CounterT*           d_histogram1[1] = {d_histogram};
-        int                 num_levels1[1]  = {num_levels};
-        LevelT*             d_levels1[1]    = {d_levels};
-
-        return MultiHistogramRange<1, 1>(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_samples,
-            d_histogram1,
-            num_levels1,
-            d_levels1,
-            num_samples,
-            1,
-            sizeof(SampleT) * num_samples,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Computes an intensity histogram from a sequence of data samples using the specified bin boundary levels.
-     *
-     * \par
-     * - A two-dimensional <em>region of interest</em> within \p d_samples can be specified
-     *   using the \p num_row_samples, num_rows, and \p row_stride_bytes parameters.
-     * - The row stride must be a whole multiple of the sample data type
-     *   size, i.e., <tt>(row_stride_bytes % sizeof(SampleT)) == 0</tt>.
-     * - The number of histogram bins is (\p num_levels - 1)
-     * - The value range for bin<sub><em>i</em></sub> is [<tt>level[i]</tt>, <tt>level[i+1]</tt>)
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the computation of a six-bin histogram
-     * from a 2x5 region of interest within a flattened 2x7 array of float samples.
-     *
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_histogram.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input samples and
-     * // output histogram
-     * int      num_row_samples;    // e.g., 5
-     * int      num_rows;           // e.g., 2;
-     * int      row_stride_bytes;   // e.g., 7 * sizeof(float)
-     * float*   d_samples;          // e.g., [2.2, 6.0, 7.1, 2.9, 3.5,   -, -,
-     *                              //        0.3, 2.9, 2.0, 6.1, 999.5, -, -]
-     * int*     d_histogram;        // e.g., [ , , , , , , , ]
-     * int      num_levels          // e.g., 7 (seven level boundaries for six bins)
-     * float    *d_levels;          // e.g., [0.0, 2.0, 4.0, 6.0, 8.0, 12.0, 16.0]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void*    d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceHistogram::HistogramRange(d_temp_storage, temp_storage_bytes,
-     *     d_samples, d_histogram, num_levels, d_levels,
-     *     num_row_samples, num_rows, row_stride_bytes);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Compute histograms
-     * cub::DeviceHistogram::HistogramRange(d_temp_storage, temp_storage_bytes,
-     *     d_samples, d_histogram, num_levels, d_levels,
-     *     num_row_samples, num_rows, row_stride_bytes);
-     *
-     * // d_histogram   <-- [1, 0, 5, 0, 3, 0, 0, 0];
-     *
-     * \endcode
-     *
-     * \tparam SampleIteratorT          <b>[inferred]</b> Random-access input iterator type for reading input samples. \iterator
-     * \tparam CounterT                 <b>[inferred]</b> Integer type for histogram bin counters
-     * \tparam LevelT                   <b>[inferred]</b> Type for specifying boundaries (levels)
-     * \tparam OffsetT                  <b>[inferred]</b> Signed integer type for sequence offsets, list lengths, pointer differences, etc.  \offset_size1
-     */
-    template <
-        typename            SampleIteratorT,
-        typename            CounterT,
-        typename            LevelT,
-        typename            OffsetT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t HistogramRange(
-        void*               d_temp_storage,                         ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&             temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        SampleIteratorT     d_samples,                              ///< [in] The pointer to the input sequence of data samples.
-        CounterT*           d_histogram,                            ///< [out] The pointer to the histogram counter output array of length <tt>num_levels</tt> - 1.
-        int                 num_levels,                             ///< [in] The number of boundaries (levels) for delineating histogram samples.  Implies that the number of bins is <tt>num_levels</tt> - 1.
-        LevelT*             d_levels,                               ///< [in] The pointer to the array of boundaries (levels).  Bin ranges are defined by consecutive boundary pairings: lower sample value boundaries are inclusive and upper sample value boundaries are exclusive.
-        OffsetT             num_row_samples,                        ///< [in] The number of data samples per row in the region of interest
-        OffsetT             num_rows,                               ///< [in] The number of rows in the region of interest
-        size_t              row_stride_bytes,                       ///< [in] The number of bytes between starts of consecutive rows in the region of interest
-        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        CounterT*           d_histogram1[1]     = {d_histogram};
-        int                 num_levels1[1]      = {num_levels};
-        LevelT*             d_levels1[1]        = {d_levels};
-
-        return MultiHistogramRange<1, 1>(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_samples,
-            d_histogram1,
-            num_levels1,
-            d_levels1,
-            num_row_samples,
-            num_rows,
-            row_stride_bytes,
-            stream,
-            debug_synchronous);
-    }
-
-    /**
-     * \brief Computes per-channel intensity histograms from a sequence of multi-channel "pixel" data samples using the specified bin boundary levels.
-     *
-     * \par
-     * - The input is a sequence of <em>pixel</em> structures, where each pixel comprises
-     *   a record of \p NUM_CHANNELS consecutive data samples (e.g., an <em>RGBA</em> pixel).
-     * - Of the \p NUM_CHANNELS specified, the function will only compute histograms
-     *   for the first \p NUM_ACTIVE_CHANNELS (e.g., <em>RGB</em> histograms from <em>RGBA</em>
-     *   pixel samples).
-     * - The number of histogram bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
-     * - For channel<sub><em>i</em></sub>, the range of values for all histogram bins
-     *   have the same width: (<tt>upper_level[i]</tt> - <tt>lower_level[i]</tt>) / (<tt> num_levels[i]</tt> - 1)
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the computation of three 4-bin <em>RGB</em> histograms
-     * from a quad-channel sequence of <em>RGBA</em> pixels (8 bits per channel per pixel)
-     *
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_histogram.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input samples
-     * // and output histograms
-     * int            num_pixels;       // e.g., 5
-     * unsigned char  *d_samples;       // e.g., [(2, 6, 7, 5),(3, 0, 2, 1),(7, 0, 6, 2),
-     *                                  //        (0, 6, 7, 5),(3, 0, 2, 6)]
-     * unsigned int   *d_histogram[3];  // e.g., [[ -, -, -, -],[ -, -, -, -],[ -, -, -, -]];
-     * int            num_levels[3];    // e.g., {5, 5, 5};
-     * unsigned int   *d_levels[3];     // e.g., [ [0, 2, 4, 6, 8],
-     *                                  //         [0, 2, 4, 6, 8],
-     *                                  //         [0, 2, 4, 6, 8] ];
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void*    d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceHistogram::MultiHistogramRange<4, 3>(d_temp_storage, temp_storage_bytes,
-     *     d_samples, d_histogram, num_levels, d_levels, num_pixels);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Compute histograms
-     * cub::DeviceHistogram::MultiHistogramRange<4, 3>(d_temp_storage, temp_storage_bytes,
-     *     d_samples, d_histogram, num_levels, d_levels, num_pixels);
-     *
-     * // d_histogram   <-- [ [1, 3, 0, 1],
-     * //                     [3, 0, 0, 2],
-     * //                     [0, 2, 0, 3] ]
-     *
-     * \endcode
-     *
-     * \tparam NUM_CHANNELS             Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed)
-     * \tparam NUM_ACTIVE_CHANNELS      <b>[inferred]</b> Number of channels actively being histogrammed
-     * \tparam SampleIteratorT          <b>[inferred]</b> Random-access input iterator type for reading input samples. \iterator
-     * \tparam CounterT                 <b>[inferred]</b> Integer type for histogram bin counters
-     * \tparam LevelT                   <b>[inferred]</b> Type for specifying boundaries (levels)
-     * \tparam OffsetT                  <b>[inferred]</b> Signed integer type for sequence offsets, list lengths, pointer differences, etc.  \offset_size1
-     */
-    template <
-        int                 NUM_CHANNELS,
-        int                 NUM_ACTIVE_CHANNELS,
-        typename            SampleIteratorT,
-        typename            CounterT,
-        typename            LevelT,
-        typename            OffsetT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t MultiHistogramRange(
-        void*               d_temp_storage,                         ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&             temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        SampleIteratorT     d_samples,                              ///< [in] The pointer to the multi-channel input sequence of data samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four <em>RGBA</em> 8-bit samples).
-        CounterT*           d_histogram[NUM_ACTIVE_CHANNELS],       ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histogram[i]</tt> should be <tt>num_levels[i]</tt> - 1.
-        int                 num_levels[NUM_ACTIVE_CHANNELS],        ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
-        LevelT*             d_levels[NUM_ACTIVE_CHANNELS],          ///< [in] The pointers to the arrays of boundaries (levels), one for each active channel.  Bin ranges are defined by consecutive boundary pairings: lower sample value boundaries are inclusive and upper sample value boundaries are exclusive.
-        OffsetT             num_pixels,                             ///< [in] The number of multi-channel pixels (i.e., the length of \p d_samples / NUM_CHANNELS)
-        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        /// The sample value type of the input iterator
-        typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
-
-        return MultiHistogramRange<NUM_CHANNELS, NUM_ACTIVE_CHANNELS>(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_samples,
-            d_histogram,
-            num_levels,
-            d_levels,
-            num_pixels,
-            1,
-            sizeof(SampleT) * NUM_CHANNELS * num_pixels,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Computes per-channel intensity histograms from a sequence of multi-channel "pixel" data samples using the specified bin boundary levels.
-     *
-     * \par
-     * - The input is a sequence of <em>pixel</em> structures, where each pixel comprises
-     *   a record of \p NUM_CHANNELS consecutive data samples (e.g., an <em>RGBA</em> pixel).
-     * - Of the \p NUM_CHANNELS specified, the function will only compute histograms
-     *   for the first \p NUM_ACTIVE_CHANNELS (e.g., <em>RGB</em> histograms from <em>RGBA</em>
-     *   pixel samples).
-     * - A two-dimensional <em>region of interest</em> within \p d_samples can be specified
-     *   using the \p num_row_samples, num_rows, and \p row_stride_bytes parameters.
-     * - The row stride must be a whole multiple of the sample data type
-     *   size, i.e., <tt>(row_stride_bytes % sizeof(SampleT)) == 0</tt>.
-     * - The number of histogram bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
-     * - For channel<sub><em>i</em></sub>, the range of values for all histogram bins
-     *   have the same width: (<tt>upper_level[i]</tt> - <tt>lower_level[i]</tt>) / (<tt> num_levels[i]</tt> - 1)
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the computation of three 4-bin <em>RGB</em> histograms from a 2x3 region of
-     * interest of within a flattened 2x4 array of quad-channel <em>RGBA</em> pixels (8 bits per channel per pixel).
-     *
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_histogram.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input samples
-     * // and output histograms
-     * int              num_row_pixels;     // e.g., 3
-     * int              num_rows;           // e.g., 2
-     * size_t           row_stride_bytes;   // e.g., 4 * sizeof(unsigned char) * NUM_CHANNELS
-     * unsigned char*   d_samples;          // e.g., [(2, 6, 7, 5),(3, 0, 2, 1),(1, 1, 1, 1),(-, -, -, -),
-     *                                      //        (7, 0, 6, 2),(0, 6, 7, 5),(3, 0, 2, 6),(-, -, -, -)]
-     * int*             d_histogram[3];     // e.g., [[ -, -, -, -],[ -, -, -, -],[ -, -, -, -]];
-     * int              num_levels[3];      // e.g., {5, 5, 5};
-     * unsigned int*    d_levels[3];        // e.g., [ [0, 2, 4, 6, 8],
-     *                                      //         [0, 2, 4, 6, 8],
-     *                                      //         [0, 2, 4, 6, 8] ];
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void*    d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceHistogram::MultiHistogramRange<4, 3>(d_temp_storage, temp_storage_bytes,
-     *     d_samples, d_histogram, num_levels, d_levels, num_row_pixels, num_rows, row_stride_bytes);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Compute histograms
-     * cub::DeviceHistogram::MultiHistogramRange<4, 3>(d_temp_storage, temp_storage_bytes,
-     *     d_samples, d_histogram, num_levels, d_levels, num_row_pixels, num_rows, row_stride_bytes);
-     *
-     * // d_histogram   <-- [ [2, 3, 0, 1],
-     * //                     [3, 0, 0, 2],
-     * //                     [1, 2, 0, 3] ]
-     *
-     * \endcode
-     *
-     * \tparam NUM_CHANNELS             Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed)
-     * \tparam NUM_ACTIVE_CHANNELS      <b>[inferred]</b> Number of channels actively being histogrammed
-     * \tparam SampleIteratorT          <b>[inferred]</b> Random-access input iterator type for reading input samples. \iterator
-     * \tparam CounterT                 <b>[inferred]</b> Integer type for histogram bin counters
-     * \tparam LevelT                   <b>[inferred]</b> Type for specifying boundaries (levels)
-     * \tparam OffsetT                  <b>[inferred]</b> Signed integer type for sequence offsets, list lengths, pointer differences, etc.  \offset_size1
-     */
-    template <
-        int                 NUM_CHANNELS,
-        int                 NUM_ACTIVE_CHANNELS,
-        typename            SampleIteratorT,
-        typename            CounterT,
-        typename            LevelT,
-        typename            OffsetT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t MultiHistogramRange(
-        void*               d_temp_storage,                         ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&             temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        SampleIteratorT     d_samples,                              ///< [in] The pointer to the multi-channel input sequence of data samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four <em>RGBA</em> 8-bit samples).
-        CounterT*           d_histogram[NUM_ACTIVE_CHANNELS],       ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histogram[i]</tt> should be <tt>num_levels[i]</tt> - 1.
-        int                 num_levels[NUM_ACTIVE_CHANNELS],        ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_levels[i]</tt> - 1.
-        LevelT*             d_levels[NUM_ACTIVE_CHANNELS],          ///< [in] The pointers to the arrays of boundaries (levels), one for each active channel.  Bin ranges are defined by consecutive boundary pairings: lower sample value boundaries are inclusive and upper sample value boundaries are exclusive.
-        OffsetT             num_row_pixels,                         ///< [in] The number of multi-channel pixels per row in the region of interest
-        OffsetT             num_rows,                               ///< [in] The number of rows in the region of interest
-        size_t              row_stride_bytes,                       ///< [in] The number of bytes between starts of consecutive rows in the region of interest
-        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        /// The sample value type of the input iterator
-        typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
-        Int2Type<sizeof(SampleT) == 1> is_byte_sample;
-
-        if ((sizeof(OffsetT) > sizeof(int)) &&
-            ((unsigned long long) (num_rows * row_stride_bytes) < (unsigned long long) std::numeric_limits<int>::max()))
-        {
-            // Down-convert OffsetT data type
-            return DipatchHistogram<NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, LevelT, int>::DispatchRange(
-                d_temp_storage, temp_storage_bytes, d_samples, d_histogram, num_levels, d_levels,
-                (int) num_row_pixels, (int) num_rows, (int) (row_stride_bytes / sizeof(SampleT)),
-                stream, debug_synchronous, is_byte_sample);
-        }
-
-        return DipatchHistogram<NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, LevelT, OffsetT>::DispatchRange(
-            d_temp_storage, temp_storage_bytes, d_samples, d_histogram, num_levels, d_levels,
-            num_row_pixels, num_rows, (OffsetT) (row_stride_bytes / sizeof(SampleT)),
-            stream, debug_synchronous, is_byte_sample);
-    }
-
-
-
-    //@}  end member group
-};
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/device/device_partition.cuh b/pfsimulator/third_party/cub/device/device_partition.cuh
deleted file mode 100644
index 505354007..000000000
--- a/pfsimulator/third_party/cub/device/device_partition.cuh
+++ /dev/null
@@ -1,273 +0,0 @@
-
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::DevicePartition provides device-wide, parallel operations for partitioning sequences of data items residing within device-accessible memory.
- */
-
-#pragma once
-
-#include <stdio.h>
-#include <iterator>
-
-#include "dispatch/dispatch_select_if.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \brief DevicePartition provides device-wide, parallel operations for partitioning sequences of data items residing within device-accessible memory. ![](partition_logo.png)
- * \ingroup SingleModule
- *
- * \par Overview
- * These operations apply a selection criterion to construct a partitioned output sequence from items selected/unselected from
- * a specified input sequence.
- *
- * \par Usage Considerations
- * \cdp_class{DevicePartition}
- *
- * \par Performance
- * \linear_performance{partition}
- *
- * \par
- * The following chart illustrates DevicePartition::If
- * performance across different CUDA architectures for \p int32 items,
- * where 50% of the items are randomly selected for the first partition.
- * \plots_below
- *
- * \image html partition_if_int32_50_percent.png
- *
- */
-struct DevicePartition
-{
-    /**
-     * \brief Uses the \p d_flags sequence to split the corresponding items from \p d_in into a partitioned sequence \p d_out.  The total number of items copied into the first partition is written to \p d_num_selected_out. ![](partition_flags_logo.png)
-     *
-     * \par
-     * - The value type of \p d_flags must be castable to \p bool (e.g., \p bool, \p char, \p int, etc.).
-     * - Copies of the selected items are compacted into \p d_out and maintain their original
-     *   relative ordering, however copies of the unselected items are compacted into the
-     *   rear of \p d_out in reverse order.
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the compaction of items selected from an \p int device vector.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>       // or equivalently <cub/device/device_partition.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input, flags, and output
-     * int  num_items;              // e.g., 8
-     * int  *d_in;                  // e.g., [1, 2, 3, 4, 5, 6, 7, 8]
-     * char *d_flags;               // e.g., [1, 0, 0, 1, 0, 1, 1, 0]
-     * int  *d_out;                 // e.g., [ ,  ,  ,  ,  ,  ,  ,  ]
-     * int  *d_num_selected_out;    // e.g., [ ]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DevicePartition::Flagged(d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run selection
-     * cub::DevicePartition::Flagged(d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items);
-     *
-     * // d_out                 <-- [1, 4, 6, 7, 8, 5, 3, 2]
-     * // d_num_selected_out    <-- [4]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT       <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
-     * \tparam FlagIterator         <b>[inferred]</b> Random-access input iterator type for reading selection flags \iterator
-     * \tparam OutputIteratorT      <b>[inferred]</b> Random-access output iterator type for writing output items \iterator
-     * \tparam NumSelectedIteratorT  <b>[inferred]</b> Output iterator type for recording the number of items selected \iterator
-     */
-    template <
-        typename                    InputIteratorT,
-        typename                    FlagIterator,
-        typename                    OutputIteratorT,
-        typename                    NumSelectedIteratorT>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Flagged(
-        void*               d_temp_storage,                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                      &temp_storage_bytes,            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT              d_in,                           ///< [in] Pointer to the input sequence of data items
-        FlagIterator                d_flags,                        ///< [in] Pointer to the input sequence of selection flags
-        OutputIteratorT             d_out,                          ///< [out] Pointer to the output sequence of partitioned data items
-        NumSelectedIteratorT        d_num_selected_out,             ///< [out] Pointer to the output total number of items selected (i.e., the offset of the unselected partition)
-        int                         num_items,                      ///< [in] Total number of items to select from
-        cudaStream_t                stream             = 0,         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous  = false)     ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        typedef int                     OffsetT;         // Signed integer type for global offsets
-        typedef NullType                SelectOp;       // Selection op (not used)
-        typedef NullType                EqualityOp;     // Equality operator (not used)
-
-        return DispatchSelectIf<InputIteratorT, FlagIterator, OutputIteratorT, NumSelectedIteratorT, SelectOp, EqualityOp, OffsetT, true>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            d_flags,
-            d_out,
-            d_num_selected_out,
-            SelectOp(),
-            EqualityOp(),
-            num_items,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Uses the \p select_op functor to split the corresponding items from \p d_in into a partitioned sequence \p d_out.  The total number of items copied into the first partition is written to \p d_num_selected_out. ![](partition_logo.png)
-     *
-     * \par
-     * - Copies of the selected items are compacted into \p d_out and maintain their original
-     *   relative ordering, however copies of the unselected items are compacted into the
-     *   rear of \p d_out in reverse order.
-     * - \devicestorage
-     *
-     * \par Performance
-     * The following charts illustrate saturated partition-if performance across different
-     * CUDA architectures for \p int32 and \p int64 items, respectively.  Items are
-     * selected for the first partition with 50% probability.
-     *
-     * \image html partition_if_int32_50_percent.png
-     * \image html partition_if_int64_50_percent.png
-     *
-     * \par
-     * The following charts are similar, but 5% selection probability for the first partition:
-     *
-     * \image html partition_if_int32_5_percent.png
-     * \image html partition_if_int64_5_percent.png
-     *
-     * \par Snippet
-     * The code snippet below illustrates the compaction of items selected from an \p int device vector.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_partition.cuh>
-     *
-     * // Functor type for selecting values less than some criteria
-     * struct LessThan
-     * {
-     *     int compare;
-     *
-     *     CUB_RUNTIME_FUNCTION __forceinline__
-     *     LessThan(int compare) : compare(compare) {}
-     *
-     *     CUB_RUNTIME_FUNCTION __forceinline__
-     *     bool operator()(const int &a) const {
-     *         return (a < compare);
-     *     }
-     * };
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int      num_items;              // e.g., 8
-     * int      *d_in;                  // e.g., [0, 2, 3, 9, 5, 2, 81, 8]
-     * int      *d_out;                 // e.g., [ ,  ,  ,  ,  ,  ,  ,  ]
-     * int      *d_num_selected_out;    // e.g., [ ]
-     * LessThan select_op(7);
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSelect::If(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, select_op);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run selection
-     * cub::DeviceSelect::If(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, select_op);
-     *
-     * // d_out                 <-- [0, 2, 3, 5, 2, 8, 81, 9]
-     * // d_num_selected_out    <-- [5]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT       <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
-     * \tparam OutputIteratorT      <b>[inferred]</b> Random-access output iterator type for writing output items \iterator
-     * \tparam NumSelectedIteratorT  <b>[inferred]</b> Output iterator type for recording the number of items selected \iterator
-     * \tparam SelectOp             <b>[inferred]</b> Selection functor type having member <tt>bool operator()(const T &a)</tt>
-     */
-    template <
-        typename                    InputIteratorT,
-        typename                    OutputIteratorT,
-        typename                    NumSelectedIteratorT,
-        typename                    SelectOp>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t If(
-        void*               d_temp_storage,                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                      &temp_storage_bytes,            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT              d_in,                           ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT             d_out,                          ///< [out] Pointer to the output sequence of partitioned data items
-        NumSelectedIteratorT        d_num_selected_out,             ///< [out] Pointer to the output total number of items selected (i.e., the offset of the unselected partition)
-        int                         num_items,                      ///< [in] Total number of items to select from
-        SelectOp                    select_op,                      ///< [in] Unary selection operator
-        cudaStream_t                stream             = 0,         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous  = false)     ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        typedef int                     OffsetT;         // Signed integer type for global offsets
-        typedef NullType*               FlagIterator;   // FlagT iterator type (not used)
-        typedef NullType                EqualityOp;     // Equality operator (not used)
-
-        return DispatchSelectIf<InputIteratorT, FlagIterator, OutputIteratorT, NumSelectedIteratorT, SelectOp, EqualityOp, OffsetT, true>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            NULL,
-            d_out,
-            d_num_selected_out,
-            select_op,
-            EqualityOp(),
-            num_items,
-            stream,
-            debug_synchronous);
-    }
-
-};
-
-/**
- * \example example_device_partition_flagged.cu
- * \example example_device_partition_if.cu
- */
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/device/device_radix_sort.cuh b/pfsimulator/third_party/cub/device/device_radix_sort.cuh
deleted file mode 100644
index 1c0bdbea1..000000000
--- a/pfsimulator/third_party/cub/device/device_radix_sort.cuh
+++ /dev/null
@@ -1,797 +0,0 @@
-
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::DeviceRadixSort provides device-wide, parallel operations for computing a radix sort across a sequence of data items residing within device-accessible memory.
- */
-
-#pragma once
-
-#include <stdio.h>
-#include <iterator>
-
-#include "dispatch/dispatch_radix_sort.cuh"
-#include "../util_arch.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \brief DeviceRadixSort provides device-wide, parallel operations for computing a radix sort across a sequence of data items residing within device-accessible memory. ![](sorting_logo.png)
- * \ingroup SingleModule
- *
- * \par Overview
- * The [<em>radix sorting method</em>](http://en.wikipedia.org/wiki/Radix_sort) arranges
- * items into ascending (or descending) order.  The algorithm relies upon a positional representation for
- * keys, i.e., each key is comprised of an ordered sequence of symbols (e.g., digits,
- * characters, etc.) specified from least-significant to most-significant.  For a
- * given input sequence of keys and a set of rules specifying a total ordering
- * of the symbolic alphabet, the radix sorting method produces a lexicographic
- * ordering of those keys.
- *
- * \par
- * DeviceRadixSort can sort all of the built-in C++ numeric primitive types
- * (<tt>unsigned char</tt>, \p int, \p double, etc.) as well as CUDA's \p __half
- * half-precision floating-point type.  Although the direct radix sorting
- * method can only be applied to unsigned integral types, DeviceRadixSort
- * is able to sort signed and floating-point types via simple bit-wise transformations
- * that ensure lexicographic key ordering.
- *
- * \par Usage Considerations
- * \cdp_class{DeviceRadixSort}
- *
- * \par Performance
- * \linear_performance{radix sort} The following chart illustrates DeviceRadixSort::SortKeys
- * performance across different CUDA architectures for uniform-random \p uint32 keys.
- * \plots_below
- *
- * \image html lsb_radix_sort_int32_keys.png
- *
- */
-struct DeviceRadixSort
-{
-
-    /******************************************************************//**
-     * \name KeyT-value pairs
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Sorts key-value pairs into ascending order. (~<em>2N </em>auxiliary storage required)
-     *
-     * \par
-     * - The contents of the input data are not altered by the sorting operation
-     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
-     * - \devicestorageNP  For sorting using only <em>O</em>(<tt>P</tt>) temporary storage, see the sorting interface using DoubleBuffer wrappers below.
-     * - \devicestorage
-     *
-     * \par Performance
-     * The following charts illustrate saturated sorting performance across different
-     * CUDA architectures for uniform-random <tt>uint32,uint32</tt> and
-     * <tt>uint64,uint64</tt> pairs, respectively.
-     *
-     * \image html lsb_radix_sort_int32_pairs.png
-     * \image html lsb_radix_sort_int64_pairs.png
-     *
-     * \par Snippet
-     * The code snippet below illustrates the sorting of a device vector of \p int keys
-     * with associated vector of \p int values.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for sorting data
-     * int  num_items;          // e.g., 7
-     * int  *d_keys_in;         // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_keys_out;        // e.g., [        ...        ]
-     * int  *d_values_in;       // e.g., [0, 1, 2, 3, 4, 5, 6]
-     * int  *d_values_out;      // e.g., [        ...        ]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceRadixSort::SortPairs(d_temp_storage, temp_storage_bytes,
-     *     d_keys_in, d_keys_out, d_values_in, d_values_out, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run sorting operation
-     * cub::DeviceRadixSort::SortPairs(d_temp_storage, temp_storage_bytes,
-     *     d_keys_in, d_keys_out, d_values_in, d_values_out, num_items);
-     *
-     * // d_keys_out            <-- [0, 3, 5, 6, 7, 8, 9]
-     * // d_values_out          <-- [5, 4, 3, 1, 2, 0, 6]
-     *
-     * \endcode
-     *
-     * \tparam KeyT      <b>[inferred]</b> KeyT type
-     * \tparam ValueT    <b>[inferred]</b> ValueT type
-     */
-    template <
-        typename            KeyT,
-        typename            ValueT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t SortPairs(
-        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        const KeyT          *d_keys_in,                             ///< [in] Pointer to the input data of key data to sort
-        KeyT                *d_keys_out,                            ///< [out] Pointer to the sorted output sequence of key data
-        const ValueT        *d_values_in,                           ///< [in] Pointer to the corresponding input sequence of associated value items
-        ValueT              *d_values_out,                          ///< [out] Pointer to the correspondingly-reordered output sequence of associated value items
-        int                 num_items,                              ///< [in] Number of items to sort
-        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
-        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
-        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        DoubleBuffer<KeyT>       d_keys(const_cast<KeyT*>(d_keys_in), d_keys_out);
-        DoubleBuffer<ValueT>     d_values(const_cast<ValueT*>(d_values_in), d_values_out);
-
-        return DispatchRadixSort<false, KeyT, ValueT, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_keys,
-            d_values,
-            num_items,
-            begin_bit,
-            end_bit,
-            false,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Sorts key-value pairs into ascending order. (~<em>N </em>auxiliary storage required)
-     *
-     * \par
-     * - The sorting operation is given a pair of key buffers and a corresponding
-     *   pair of associated value buffers.  Each pair is managed by a DoubleBuffer
-     *   structure that indicates which of the two buffers is "current" (and thus
-     *   contains the input data to be sorted).
-     * - The contents of both buffers within each pair may be altered by the sorting
-     *   operation.
-     * - Upon completion, the sorting operation will update the "current" indicator
-     *   within each DoubleBuffer wrapper to reference which of the two buffers
-     *   now contains the sorted output sequence (a function of the number of key bits
-     *   specified and the targeted device architecture).
-     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
-     * - \devicestorageP
-     * - \devicestorage
-     *
-     * \par Performance
-     * The following charts illustrate saturated sorting performance across different
-     * CUDA architectures for uniform-random <tt>uint32,uint32</tt> and
-     * <tt>uint64,uint64</tt> pairs, respectively.
-     *
-     * \image html lsb_radix_sort_int32_pairs.png
-     * \image html lsb_radix_sort_int64_pairs.png
-     *
-     * \par Snippet
-     * The code snippet below illustrates the sorting of a device vector of \p int keys
-     * with associated vector of \p int values.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for sorting data
-     * int  num_items;          // e.g., 7
-     * int  *d_key_buf;         // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_key_alt_buf;     // e.g., [        ...        ]
-     * int  *d_value_buf;       // e.g., [0, 1, 2, 3, 4, 5, 6]
-     * int  *d_value_alt_buf;   // e.g., [        ...        ]
-     * ...
-     *
-     * // Create a set of DoubleBuffers to wrap pairs of device pointers
-     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
-     * cub::DoubleBuffer<int> d_values(d_value_buf, d_value_alt_buf);
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, d_keys, d_values, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run sorting operation
-     * cub::DeviceRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, d_keys, d_values, num_items);
-     *
-     * // d_keys.Current()      <-- [0, 3, 5, 6, 7, 8, 9]
-     * // d_values.Current()    <-- [5, 4, 3, 1, 2, 0, 6]
-     *
-     * \endcode
-     *
-     * \tparam KeyT      <b>[inferred]</b> KeyT type
-     * \tparam ValueT    <b>[inferred]</b> ValueT type
-     */
-    template <
-        typename            KeyT,
-        typename            ValueT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t SortPairs(
-        void                    *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                  &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        DoubleBuffer<KeyT>      &d_keys,                                ///< [in,out] Reference to the double-buffer of keys whose "current" device-accessible buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
-        DoubleBuffer<ValueT>    &d_values,                              ///< [in,out] Double-buffer of values whose "current" device-accessible buffer contains the unsorted input values and, upon return, is updated to point to the sorted output values
-        int                     num_items,                              ///< [in] Number of items to sort
-        int                     begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
-        int                     end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
-        cudaStream_t            stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                    debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        return DispatchRadixSort<false, KeyT, ValueT, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_keys,
-            d_values,
-            num_items,
-            begin_bit,
-            end_bit,
-            true,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Sorts key-value pairs into descending order. (~<em>2N</em> auxiliary storage required).
-     *
-     * \par
-     * - The contents of the input data are not altered by the sorting operation
-     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
-     * - \devicestorageNP  For sorting using only <em>O</em>(<tt>P</tt>) temporary storage, see the sorting interface using DoubleBuffer wrappers below.
-     * - \devicestorage
-     *
-     * \par Performance
-     * Performance is similar to DeviceRadixSort::SortPairs.
-     *
-     * \par Snippet
-     * The code snippet below illustrates the sorting of a device vector of \p int keys
-     * with associated vector of \p int values.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for sorting data
-     * int  num_items;          // e.g., 7
-     * int  *d_keys_in;         // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_keys_out;        // e.g., [        ...        ]
-     * int  *d_values_in;       // e.g., [0, 1, 2, 3, 4, 5, 6]
-     * int  *d_values_out;      // e.g., [        ...        ]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes,
-     *     d_keys_in, d_keys_out, d_values_in, d_values_out, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run sorting operation
-     * cub::DeviceRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes,
-     *     d_keys_in, d_keys_out, d_values_in, d_values_out, num_items);
-     *
-     * // d_keys_out            <-- [9, 8, 7, 6, 5, 3, 0]
-     * // d_values_out          <-- [6, 0, 2, 1, 3, 4, 5]
-     *
-     * \endcode
-     *
-     * \tparam KeyT      <b>[inferred]</b> KeyT type
-     * \tparam ValueT    <b>[inferred]</b> ValueT type
-     */
-    template <
-        typename            KeyT,
-        typename            ValueT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t SortPairsDescending(
-        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        const KeyT          *d_keys_in,                             ///< [in] Pointer to the input data of key data to sort
-        KeyT                *d_keys_out,                            ///< [out] Pointer to the sorted output sequence of key data
-        const ValueT        *d_values_in,                           ///< [in] Pointer to the corresponding input sequence of associated value items
-        ValueT              *d_values_out,                          ///< [out] Pointer to the correspondingly-reordered output sequence of associated value items
-        int                 num_items,                              ///< [in] Number of items to sort
-        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
-        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
-        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        DoubleBuffer<KeyT>       d_keys(const_cast<KeyT*>(d_keys_in), d_keys_out);
-        DoubleBuffer<ValueT>     d_values(const_cast<ValueT*>(d_values_in), d_values_out);
-
-        return DispatchRadixSort<true, KeyT, ValueT, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_keys,
-            d_values,
-            num_items,
-            begin_bit,
-            end_bit,
-            false,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Sorts key-value pairs into descending order. (~<em>N </em>auxiliary storage required).
-     *
-     * \par
-     * - The sorting operation is given a pair of key buffers and a corresponding
-     *   pair of associated value buffers.  Each pair is managed by a DoubleBuffer
-     *   structure that indicates which of the two buffers is "current" (and thus
-     *   contains the input data to be sorted).
-     * - The contents of both buffers within each pair may be altered by the sorting
-     *   operation.
-     * - Upon completion, the sorting operation will update the "current" indicator
-     *   within each DoubleBuffer wrapper to reference which of the two buffers
-     *   now contains the sorted output sequence (a function of the number of key bits
-     *   specified and the targeted device architecture).
-     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
-     * - \devicestorageP
-     * - \devicestorage
-     *
-     * \par Performance
-     * Performance is similar to DeviceRadixSort::SortPairs.
-     *
-     * \par Snippet
-     * The code snippet below illustrates the sorting of a device vector of \p int keys
-     * with associated vector of \p int values.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for sorting data
-     * int  num_items;          // e.g., 7
-     * int  *d_key_buf;         // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_key_alt_buf;     // e.g., [        ...        ]
-     * int  *d_value_buf;       // e.g., [0, 1, 2, 3, 4, 5, 6]
-     * int  *d_value_alt_buf;   // e.g., [        ...        ]
-     * ...
-     *
-     * // Create a set of DoubleBuffers to wrap pairs of device pointers
-     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
-     * cub::DoubleBuffer<int> d_values(d_value_buf, d_value_alt_buf);
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes, d_keys, d_values, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run sorting operation
-     * cub::DeviceRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes, d_keys, d_values, num_items);
-     *
-     * // d_keys.Current()      <-- [9, 8, 7, 6, 5, 3, 0]
-     * // d_values.Current()    <-- [6, 0, 2, 1, 3, 4, 5]
-     *
-     * \endcode
-     *
-     * \tparam KeyT      <b>[inferred]</b> KeyT type
-     * \tparam ValueT    <b>[inferred]</b> ValueT type
-     */
-    template <
-        typename            KeyT,
-        typename            ValueT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t SortPairsDescending(
-        void                    *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                  &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        DoubleBuffer<KeyT>      &d_keys,                                ///< [in,out] Reference to the double-buffer of keys whose "current" device-accessible buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
-        DoubleBuffer<ValueT>    &d_values,                              ///< [in,out] Double-buffer of values whose "current" device-accessible buffer contains the unsorted input values and, upon return, is updated to point to the sorted output values
-        int                     num_items,                              ///< [in] Number of items to sort
-        int                     begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
-        int                     end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
-        cudaStream_t            stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                    debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        return DispatchRadixSort<true, KeyT, ValueT, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_keys,
-            d_values,
-            num_items,
-            begin_bit,
-            end_bit,
-            true,
-            stream,
-            debug_synchronous);
-    }
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Keys-only
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Sorts keys into ascending order. (~<em>2N </em>auxiliary storage required)
-     *
-     * \par
-     * - The contents of the input data are not altered by the sorting operation
-     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
-     * - \devicestorageNP  For sorting using only <em>O</em>(<tt>P</tt>) temporary storage, see the sorting interface using DoubleBuffer wrappers below.
-     * - \devicestorage
-     *
-     * \par Performance
-     * The following charts illustrate saturated sorting performance across different
-     * CUDA architectures for uniform-random \p uint32 and \p uint64 keys, respectively.
-     *
-     * \image html lsb_radix_sort_int32_keys.png
-     * \image html lsb_radix_sort_int64_keys.png
-     *
-     * \par Snippet
-     * The code snippet below illustrates the sorting of a device vector of \p int keys.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for sorting data
-     * int  num_items;          // e.g., 7
-     * int  *d_keys_in;         // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_keys_out;        // e.g., [        ...        ]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run sorting operation
-     * cub::DeviceRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out, num_items);
-     *
-     * // d_keys_out            <-- [0, 3, 5, 6, 7, 8, 9]
-     *
-     * \endcode
-     *
-     * \tparam KeyT      <b>[inferred]</b> KeyT type
-     */
-    template <typename KeyT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t SortKeys(
-        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        const KeyT          *d_keys_in,                             ///< [in] Pointer to the input data of key data to sort
-        KeyT                *d_keys_out,                            ///< [out] Pointer to the sorted output sequence of key data
-        int                 num_items,                              ///< [in] Number of items to sort
-        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
-        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
-        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        // Null value type
-        DoubleBuffer<KeyT>      d_keys(const_cast<KeyT*>(d_keys_in), d_keys_out);
-        DoubleBuffer<NullType>  d_values;
-
-        return DispatchRadixSort<false, KeyT, NullType, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_keys,
-            d_values,
-            num_items,
-            begin_bit,
-            end_bit,
-            false,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Sorts keys into ascending order. (~<em>N </em>auxiliary storage required).
-     *
-     * \par
-     * - The sorting operation is given a pair of key buffers managed by a
-     *   DoubleBuffer structure that indicates which of the two buffers is
-     *   "current" (and thus contains the input data to be sorted).
-     * - The contents of both buffers may be altered by the sorting operation.
-     * - Upon completion, the sorting operation will update the "current" indicator
-     *   within the DoubleBuffer wrapper to reference which of the two buffers
-     *   now contains the sorted output sequence (a function of the number of key bits
-     *   specified and the targeted device architecture).
-     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
-     * - \devicestorageP
-     * - \devicestorage
-     *
-     * \par Performance
-     * The following charts illustrate saturated sorting performance across different
-     * CUDA architectures for uniform-random \p uint32 and \p uint64 keys, respectively.
-     *
-     * \image html lsb_radix_sort_int32_keys.png
-     * \image html lsb_radix_sort_int64_keys.png
-     *
-     * \par Snippet
-     * The code snippet below illustrates the sorting of a device vector of \p int keys.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for sorting data
-     * int  num_items;          // e.g., 7
-     * int  *d_key_buf;         // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_key_alt_buf;     // e.g., [        ...        ]
-     * ...
-     *
-     * // Create a DoubleBuffer to wrap the pair of device pointers
-     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run sorting operation
-     * cub::DeviceRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys, num_items);
-     *
-     * // d_keys.Current()      <-- [0, 3, 5, 6, 7, 8, 9]
-     *
-     * \endcode
-     *
-     * \tparam KeyT      <b>[inferred]</b> KeyT type
-     */
-    template <typename KeyT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t SortKeys(
-        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        DoubleBuffer<KeyT>  &d_keys,                                ///< [in,out] Reference to the double-buffer of keys whose "current" device-accessible buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
-        int                 num_items,                              ///< [in] Number of items to sort
-        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
-        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
-        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        // Null value type
-        DoubleBuffer<NullType> d_values;
-
-        return DispatchRadixSort<false, KeyT, NullType, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_keys,
-            d_values,
-            num_items,
-            begin_bit,
-            end_bit,
-            true,
-            stream,
-            debug_synchronous);
-    }
-
-    /**
-     * \brief Sorts keys into descending order. (~<em>2N</em> auxiliary storage required).
-     *
-     * \par
-     * - The contents of the input data are not altered by the sorting operation
-     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
-     * - \devicestorageNP  For sorting using only <em>O</em>(<tt>P</tt>) temporary storage, see the sorting interface using DoubleBuffer wrappers below.
-     * - \devicestorage
-     *
-     * \par Performance
-     * Performance is similar to DeviceRadixSort::SortKeys.
-     *
-     * \par Snippet
-     * The code snippet below illustrates the sorting of a device vector of \p int keys.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for sorting data
-     * int  num_items;          // e.g., 7
-     * int  *d_keys_in;         // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_keys_out;        // e.g., [        ...        ]
-     * ...
-     *
-     * // Create a DoubleBuffer to wrap the pair of device pointers
-     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceRadixSort::SortKeysDescending(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run sorting operation
-     * cub::DeviceRadixSort::SortKeysDescending(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out, num_items);
-     *
-     * // d_keys_out            <-- [9, 8, 7, 6, 5, 3, 0]s
-     *
-     * \endcode
-     *
-     * \tparam KeyT      <b>[inferred]</b> KeyT type
-     */
-    template <typename KeyT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t SortKeysDescending(
-        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        const KeyT          *d_keys_in,                             ///< [in] Pointer to the input data of key data to sort
-        KeyT                *d_keys_out,                            ///< [out] Pointer to the sorted output sequence of key data
-        int                 num_items,                              ///< [in] Number of items to sort
-        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
-        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
-        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        DoubleBuffer<KeyT>      d_keys(const_cast<KeyT*>(d_keys_in), d_keys_out);
-        DoubleBuffer<NullType>  d_values;
-
-        return DispatchRadixSort<true, KeyT, NullType, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_keys,
-            d_values,
-            num_items,
-            begin_bit,
-            end_bit,
-            false,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Sorts keys into descending order. (~<em>N </em>auxiliary storage required).
-     *
-     * \par
-     * - The sorting operation is given a pair of key buffers managed by a
-     *   DoubleBuffer structure that indicates which of the two buffers is
-     *   "current" (and thus contains the input data to be sorted).
-     * - The contents of both buffers may be altered by the sorting operation.
-     * - Upon completion, the sorting operation will update the "current" indicator
-     *   within the DoubleBuffer wrapper to reference which of the two buffers
-     *   now contains the sorted output sequence (a function of the number of key bits
-     *   specified and the targeted device architecture).
-     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
-     * - \devicestorageP
-     * - \devicestorage
-     *
-     * \par Performance
-     * Performance is similar to DeviceRadixSort::SortKeys.
-     *
-     * \par Snippet
-     * The code snippet below illustrates the sorting of a device vector of \p int keys.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for sorting data
-     * int  num_items;          // e.g., 7
-     * int  *d_key_buf;         // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_key_alt_buf;     // e.g., [        ...        ]
-     * ...
-     *
-     * // Create a DoubleBuffer to wrap the pair of device pointers
-     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceRadixSort::SortKeysDescending(d_temp_storage, temp_storage_bytes, d_keys, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run sorting operation
-     * cub::DeviceRadixSort::SortKeysDescending(d_temp_storage, temp_storage_bytes, d_keys, num_items);
-     *
-     * // d_keys.Current()      <-- [9, 8, 7, 6, 5, 3, 0]
-     *
-     * \endcode
-     *
-     * \tparam KeyT      <b>[inferred]</b> KeyT type
-     */
-    template <typename KeyT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t SortKeysDescending(
-        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        DoubleBuffer<KeyT>  &d_keys,                                ///< [in,out] Reference to the double-buffer of keys whose "current" device-accessible buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
-        int                 num_items,                              ///< [in] Number of items to sort
-        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
-        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
-        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        // Null value type
-        DoubleBuffer<NullType> d_values;
-
-        return DispatchRadixSort<true, KeyT, NullType, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_keys,
-            d_values,
-            num_items,
-            begin_bit,
-            end_bit,
-            true,
-            stream,
-            debug_synchronous);
-    }
-
-
-    //@}  end member group
-
-
-};
-
-/**
- * \example example_device_radix_sort.cu
- */
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/device/device_reduce.cuh b/pfsimulator/third_party/cub/device/device_reduce.cuh
deleted file mode 100644
index 13c7a72d1..000000000
--- a/pfsimulator/third_party/cub/device/device_reduce.cuh
+++ /dev/null
@@ -1,734 +0,0 @@
-
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::DeviceReduce provides device-wide, parallel operations for computing a reduction across a sequence of data items residing within device-accessible memory.
- */
-
-#pragma once
-
-#include <stdio.h>
-#include <iterator>
-#include <limits>
-
-#include "../iterator/arg_index_input_iterator.cuh"
-#include "dispatch/dispatch_reduce.cuh"
-#include "dispatch/dispatch_reduce_by_key.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \brief DeviceReduce provides device-wide, parallel operations for computing a reduction across a sequence of data items residing within device-accessible memory. ![](reduce_logo.png)
- * \ingroup SingleModule
- *
- * \par Overview
- * A <a href="http://en.wikipedia.org/wiki/Reduce_(higher-order_function)"><em>reduction</em></a> (or <em>fold</em>)
- * uses a binary combining operator to compute a single aggregate from a sequence of input elements.
- *
- * \par Usage Considerations
- * \cdp_class{DeviceReduce}
- *
- * \par Performance
- * \linear_performance{reduction, reduce-by-key, and run-length encode}
- *
- * \par
- * The following chart illustrates DeviceReduce::Sum
- * performance across different CUDA architectures for \p int32 keys.
- *
- * \image html reduce_int32.png
- *
- * \par
- * The following chart illustrates DeviceReduce::ReduceByKey (summation)
- * performance across different CUDA architectures for \p fp32
- * values.  Segments are identified by \p int32 keys, and have lengths uniformly sampled from [1,1000].
- *
- * \image html reduce_by_key_fp32_len_500.png
- *
- * \par
- * \plots_below
- *
- */
-struct DeviceReduce
-{
-    /**
-     * \brief Computes a device-wide reduction using the specified binary \p reduction_op functor and initial value \p init.
-     *
-     * \par
-     * - Does not support binary reduction operators that are non-commutative.
-     * - Provides "run-to-run" determinism for pseudo-associative reduction
-     *   (e.g., addition of floating point types) on the same GPU device.
-     *   However, results for pseudo-associative reduction may be inconsistent
-     *   from one device to a another device of a different compute-capability
-     *   because CUB can employ different tile-sizing for different architectures.
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates a user-defined min-reduction of a device vector of \p int data elements.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
-     *
-     * // CustomMin functor
-     * struct CustomMin
-     * {
-     *     template <typename T>
-     *     __device__ __forceinline__
-     *     T operator()(const T &a, const T &b) const {
-     *         return (b < a) ? b : a;
-     *     }
-     * };
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int          num_items;  // e.g., 7
-     * int          *d_in;      // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int          *d_out;     // e.g., [-]
-     * CustomMin    min_op;
-     * int          init;       // e.g., INT_MAX
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceReduce::Reduce(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, min_op, init);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run reduction
-     * cub::DeviceReduce::Reduce(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, min_op, init);
-     *
-     * // d_out <-- [0]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT       <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
-     * \tparam OutputIteratorT      <b>[inferred]</b> Output iterator type for recording the reduced aggregate \iterator
-     * \tparam ReductionOpT         <b>[inferred]</b> Binary reduction functor type having member <tt>T operator()(const T &a, const T &b)</tt> 
-     * \tparam T                    <b>[inferred]</b> Data element type that is convertible to the \p value type of \p InputIteratorT
-     */
-    template <
-        typename                    InputIteratorT,
-        typename                    OutputIteratorT,
-        typename                    ReductionOpT,
-        typename                    T>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t Reduce(
-        void                        *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                      &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT              d_in,                               ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT             d_out,                              ///< [out] Pointer to the output aggregate
-        int                         num_items,                          ///< [in] Total number of input items (i.e., length of \p d_in)
-        ReductionOpT                reduction_op,                       ///< [in] Binary reduction functor
-        T                           init,                               ///< [in] Initial value of the reduction
-        cudaStream_t                stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        return DispatchReduce<InputIteratorT, OutputIteratorT, OffsetT, ReductionOpT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            d_out,
-            num_items,
-            reduction_op,
-            init,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Computes a device-wide sum using the addition (\p +) operator.
-     *
-     * \par
-     * - Uses \p 0 as the initial value of the reduction.
-     * - Does not support \p + operators that are non-commutative..
-     * - Provides "run-to-run" determinism for pseudo-associative reduction
-     *   (e.g., addition of floating point types) on the same GPU device.
-     *   However, results for pseudo-associative reduction may be inconsistent
-     *   from one device to a another device of a different compute-capability
-     *   because CUB can employ different tile-sizing for different architectures.
-     * - \devicestorage
-     *
-     * \par Performance
-     * The following charts illustrate saturated sum-reduction performance across different
-     * CUDA architectures for \p int32 and \p int64 items, respectively.
-     *
-     * \image html reduce_int32.png
-     * \image html reduce_int64.png
-     *
-     * \par Snippet
-     * The code snippet below illustrates the sum-reduction of a device vector of \p int data elements.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int  num_items;      // e.g., 7
-     * int  *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_out;         // e.g., [-]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceReduce::Sum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run sum-reduction
-     * cub::DeviceReduce::Sum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items);
-     *
-     * // d_out <-- [38]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
-     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate \iterator
-     */
-    template <
-        typename                    InputIteratorT,
-        typename                    OutputIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t Sum(
-        void                        *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                      &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT              d_in,                               ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT             d_out,                              ///< [out] Pointer to the output aggregate
-        int                         num_items,                          ///< [in] Total number of input items (i.e., length of \p d_in)
-        cudaStream_t                stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        // The output value type
-        typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
-            typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
-            typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
-
-        return DispatchReduce<InputIteratorT, OutputIteratorT, OffsetT, cub::Sum>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            d_out,
-            num_items,
-            cub::Sum(),
-            OutputT(),            // zero-initialize
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Computes a device-wide minimum using the less-than ('<') operator.
-     *
-     * \par
-     * - Uses <tt>std::numeric_limits<T>::max()</tt> as the initial value of the reduction.
-     * - Does not support \p < operators that are non-commutative.
-     * - Provides "run-to-run" determinism for pseudo-associative reduction
-     *   (e.g., addition of floating point types) on the same GPU device.
-     *   However, results for pseudo-associative reduction may be inconsistent
-     *   from one device to a another device of a different compute-capability
-     *   because CUB can employ different tile-sizing for different architectures.
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the min-reduction of a device vector of \p int data elements.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int  num_items;      // e.g., 7
-     * int  *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_out;         // e.g., [-]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceReduce::Min(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run min-reduction
-     * cub::DeviceReduce::Min(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items);
-     *
-     * // d_out <-- [0]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
-     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate \iterator
-     */
-    template <
-        typename                    InputIteratorT,
-        typename                    OutputIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t Min(
-        void                        *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                      &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT              d_in,                               ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT             d_out,                              ///< [out] Pointer to the output aggregate
-        int                         num_items,                          ///< [in] Total number of input items (i.e., length of \p d_in)
-        cudaStream_t                stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        // The input value type
-        typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
-
-        return DispatchReduce<InputIteratorT, OutputIteratorT, OffsetT, cub::Min>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            d_out,
-            num_items,
-            cub::Min(),
-            Traits<InputT>::Max(), // replace with std::numeric_limits<T>::max() when C++11 support is more prevalent
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Finds the first device-wide minimum using the less-than ('<') operator, also returning the index of that item.
-     *
-     * \par
-     * - The output value type of \p d_out is cub::KeyValuePair <tt><int, T></tt> (assuming the value type of \p d_in is \p T)
-     *   - The minimum is written to <tt>d_out.value</tt> and its offset in the input array is written to <tt>d_out.key</tt>.
-     *   - The <tt>{1, std::numeric_limits<T>::max()}</tt> tuple is produced for zero-length inputs
-     * - Does not support \p < operators that are non-commutative.
-     * - Provides "run-to-run" determinism for pseudo-associative reduction
-     *   (e.g., addition of floating point types) on the same GPU device.
-     *   However, results for pseudo-associative reduction may be inconsistent
-     *   from one device to a another device of a different compute-capability
-     *   because CUB can employ different tile-sizing for different architectures.
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the argmin-reduction of a device vector of \p int data elements.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int                      num_items;      // e.g., 7
-     * int                      *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * KeyValuePair<int, int>   *d_out;         // e.g., [{-,-}]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceReduce::ArgMin(d_temp_storage, temp_storage_bytes, d_in, d_argmin, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run argmin-reduction
-     * cub::DeviceReduce::ArgMin(d_temp_storage, temp_storage_bytes, d_in, d_argmin, num_items);
-     *
-     * // d_out <-- [{5, 0}]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items (of some type \p T) \iterator
-     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate (having value type <tt>cub::KeyValuePair<int, T></tt>) \iterator
-     */
-    template <
-        typename                    InputIteratorT,
-        typename                    OutputIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t ArgMin(
-        void                        *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                      &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT              d_in,                               ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT             d_out,                              ///< [out] Pointer to the output aggregate
-        int                         num_items,                          ///< [in] Total number of input items (i.e., length of \p d_in)
-        cudaStream_t                stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        // The input type
-        typedef typename std::iterator_traits<InputIteratorT>::value_type InputValueT;
-
-        // The output tuple type
-        typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
-            KeyValuePair<OffsetT, InputValueT>,                                                                 // ... then the key value pair OffsetT + InputValueT
-            typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputTupleT;                     // ... else the output iterator's value type
-
-        // The output value type
-        typedef typename OutputTupleT::Value OutputValueT;
-
-        // Wrapped input iterator to produce index-value <OffsetT, InputT> tuples
-        typedef ArgIndexInputIterator<InputIteratorT, OffsetT, OutputValueT> ArgIndexInputIteratorT;
-        ArgIndexInputIteratorT d_indexed_in(d_in);
-
-        // Initial value
-        OutputTupleT initial_value(1, Traits<InputValueT>::Max());   // replace with std::numeric_limits<T>::max() when C++11 support is more prevalent
-
-        return DispatchReduce<ArgIndexInputIteratorT, OutputIteratorT, OffsetT, cub::ArgMin>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_indexed_in,
-            d_out,
-            num_items,
-            cub::ArgMin(),
-            initial_value,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Computes a device-wide maximum using the greater-than ('>') operator.
-     *
-     * \par
-     * - Uses <tt>std::numeric_limits<T>::lowest()</tt> as the initial value of the reduction.
-     * - Does not support \p > operators that are non-commutative.
-     * - Provides "run-to-run" determinism for pseudo-associative reduction
-     *   (e.g., addition of floating point types) on the same GPU device.
-     *   However, results for pseudo-associative reduction may be inconsistent
-     *   from one device to a another device of a different compute-capability
-     *   because CUB can employ different tile-sizing for different architectures.
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the max-reduction of a device vector of \p int data elements.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int  num_items;      // e.g., 7
-     * int  *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_out;         // e.g., [-]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceReduce::Max(d_temp_storage, temp_storage_bytes, d_in, d_max, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run max-reduction
-     * cub::DeviceReduce::Max(d_temp_storage, temp_storage_bytes, d_in, d_max, num_items);
-     *
-     * // d_out <-- [9]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
-     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate \iterator
-     */
-    template <
-        typename                    InputIteratorT,
-        typename                    OutputIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t Max(
-        void                        *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                      &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT              d_in,                               ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT             d_out,                              ///< [out] Pointer to the output aggregate
-        int                         num_items,                          ///< [in] Total number of input items (i.e., length of \p d_in)
-        cudaStream_t                stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        // The input value type
-        typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
-
-        return DispatchReduce<InputIteratorT, OutputIteratorT, OffsetT, cub::Max>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            d_out,
-            num_items,
-            cub::Max(),
-            Traits<InputT>::Lowest(),    // replace with std::numeric_limits<T>::lowest() when C++11 support is more prevalent
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Finds the first device-wide maximum using the greater-than ('>') operator, also returning the index of that item
-     *
-     * \par
-     * - The output value type of \p d_out is cub::KeyValuePair <tt><int, T></tt> (assuming the value type of \p d_in is \p T)
-     *   - The maximum is written to <tt>d_out.value</tt> and its offset in the input array is written to <tt>d_out.key</tt>.
-     *   - The <tt>{1, std::numeric_limits<T>::lowest()}</tt> tuple is produced for zero-length inputs
-     * - Does not support \p > operators that are non-commutative.
-     * - Provides "run-to-run" determinism for pseudo-associative reduction
-     *   (e.g., addition of floating point types) on the same GPU device.
-     *   However, results for pseudo-associative reduction may be inconsistent
-     *   from one device to a another device of a different compute-capability
-     *   because CUB can employ different tile-sizing for different architectures.
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the argmax-reduction of a device vector of \p int data elements.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_reduce.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int                      num_items;      // e.g., 7
-     * int                      *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * KeyValuePair<int, int>   *d_out;         // e.g., [{-,-}]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceReduce::ArgMax(d_temp_storage, temp_storage_bytes, d_in, d_argmax, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run argmax-reduction
-     * cub::DeviceReduce::ArgMax(d_temp_storage, temp_storage_bytes, d_in, d_argmax, num_items);
-     *
-     * // d_out <-- [{6, 9}]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items (of some type \p T) \iterator
-     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate (having value type <tt>cub::KeyValuePair<int, T></tt>) \iterator
-     */
-    template <
-        typename                    InputIteratorT,
-        typename                    OutputIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t ArgMax(
-        void                        *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                      &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT              d_in,                               ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT             d_out,                              ///< [out] Pointer to the output aggregate
-        int                         num_items,                          ///< [in] Total number of input items (i.e., length of \p d_in)
-        cudaStream_t                stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        // The input type
-        typedef typename std::iterator_traits<InputIteratorT>::value_type InputValueT;
-
-        // The output tuple type
-        typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
-            KeyValuePair<OffsetT, InputValueT>,                                                                 // ... then the key value pair OffsetT + InputValueT
-            typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputTupleT;                     // ... else the output iterator's value type
-
-        // The output value type
-        typedef typename OutputTupleT::Value OutputValueT;
-
-        // Wrapped input iterator to produce index-value <OffsetT, InputT> tuples
-        typedef ArgIndexInputIterator<InputIteratorT, OffsetT, OutputValueT> ArgIndexInputIteratorT;
-        ArgIndexInputIteratorT d_indexed_in(d_in);
-
-        // Initial value
-        OutputTupleT initial_value(1, Traits<InputValueT>::Lowest());     // replace with std::numeric_limits<T>::lowest() when C++11 support is more prevalent
-
-        return DispatchReduce<ArgIndexInputIteratorT, OutputIteratorT, OffsetT, cub::ArgMax>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_indexed_in,
-            d_out,
-            num_items,
-            cub::ArgMax(),
-            initial_value,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Reduces segments of values, where segments are demarcated by corresponding runs of identical keys.
-     *
-     * \par
-     * This operation computes segmented reductions within \p d_values_in using
-     * the specified binary \p reduction_op functor.  The segments are identified by
-     * "runs" of corresponding keys in \p d_keys_in, where runs are maximal ranges of
-     * consecutive, identical keys.  For the <em>i</em><sup>th</sup> run encountered,
-     * the first key of the run and the corresponding value aggregate of that run are
-     * written to <tt>d_unique_out[<em>i</em>]</tt> and <tt>d_aggregates_out[<em>i</em>]</tt>,
-     * respectively. The total number of runs encountered is written to \p d_num_runs_out.
-     *
-     * \par
-     * - The <tt>==</tt> equality operator is used to determine whether keys are equivalent
-     * - Provides "run-to-run" determinism for pseudo-associative reduction
-     *   (e.g., addition of floating point types) on the same GPU device.
-     *   However, results for pseudo-associative reduction may be inconsistent
-     *   from one device to a another device of a different compute-capability
-     *   because CUB can employ different tile-sizing for different architectures.
-     * - \devicestorage
-     *
-     * \par Performance
-     * The following chart illustrates reduction-by-key (sum) performance across
-     * different CUDA architectures for \p fp32 and \p fp64 values, respectively.  Segments
-     * are identified by \p int32 keys, and have lengths uniformly sampled from [1,1000].
-     *
-     * \image html reduce_by_key_fp32_len_500.png
-     * \image html reduce_by_key_fp64_len_500.png
-     *
-     * \par
-     * The following charts are similar, but with segment lengths uniformly sampled from [1,10]:
-     *
-     * \image html reduce_by_key_fp32_len_5.png
-     * \image html reduce_by_key_fp64_len_5.png
-     *
-     * \par Snippet
-     * The code snippet below illustrates the segmented reduction of \p int values grouped
-     * by runs of associated \p int keys.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_reduce.cuh>
-     *
-     * // CustomMin functor
-     * struct CustomMin
-     * {
-     *     template <typename T>
-     *     CUB_RUNTIME_FUNCTION __forceinline__
-     *     T operator()(const T &a, const T &b) const {
-     *         return (b < a) ? b : a;
-     *     }
-     * };
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int          num_items;          // e.g., 8
-     * int          *d_keys_in;         // e.g., [0, 2, 2, 9, 5, 5, 5, 8]
-     * int          *d_values_in;       // e.g., [0, 7, 1, 6, 2, 5, 3, 4]
-     * int          *d_unique_out;      // e.g., [-, -, -, -, -, -, -, -]
-     * int          *d_aggregates_out;  // e.g., [-, -, -, -, -, -, -, -]
-     * int          *d_num_runs_out;    // e.g., [-]
-     * CustomMin    reduction_op;
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceReduce::ReduceByKey(d_temp_storage, temp_storage_bytes, d_keys_in, d_unique_out, d_values_in, d_aggregates_out, d_num_runs_out, reduction_op, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run reduce-by-key
-     * cub::DeviceReduce::ReduceByKey(d_temp_storage, temp_storage_bytes, d_keys_in, d_unique_out, d_values_in, d_aggregates_out, d_num_runs_out, reduction_op, num_items);
-     *
-     * // d_unique_out      <-- [0, 2, 9, 5, 8]
-     * // d_aggregates_out  <-- [0, 1, 6, 2, 4]
-     * // d_num_runs_out    <-- [5]
-     *
-     * \endcode
-     *
-     * \tparam KeysInputIteratorT       <b>[inferred]</b> Random-access input iterator type for reading input keys \iterator
-     * \tparam UniqueOutputIteratorT    <b>[inferred]</b> Random-access output iterator type for writing unique output keys \iterator
-     * \tparam ValuesInputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input values \iterator
-     * \tparam AggregatesOutputIterator <b>[inferred]</b> Random-access output iterator type for writing output value aggregates \iterator
-     * \tparam NumRunsOutputIteratorT   <b>[inferred]</b> Output iterator type for recording the number of runs encountered \iterator
-     * \tparam ReductionOpT              <b>[inferred]</b> Binary reduction functor type having member <tt>T operator()(const T &a, const T &b)</tt> 
-     */
-    template <
-        typename                    KeysInputIteratorT,
-        typename                    UniqueOutputIteratorT,
-        typename                    ValuesInputIteratorT,
-        typename                    AggregatesOutputIteratorT,
-        typename                    NumRunsOutputIteratorT,
-        typename                    ReductionOpT>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t ReduceByKey(
-        void                        *d_temp_storage,                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                      &temp_storage_bytes,            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        KeysInputIteratorT          d_keys_in,                      ///< [in] Pointer to the input sequence of keys
-        UniqueOutputIteratorT       d_unique_out,                   ///< [out] Pointer to the output sequence of unique keys (one key per run)
-        ValuesInputIteratorT        d_values_in,                    ///< [in] Pointer to the input sequence of corresponding values
-        AggregatesOutputIteratorT   d_aggregates_out,               ///< [out] Pointer to the output sequence of value aggregates (one aggregate per run)
-        NumRunsOutputIteratorT      d_num_runs_out,                 ///< [out] Pointer to total number of runs encountered (i.e., the length of d_unique_out)
-        ReductionOpT                reduction_op,                   ///< [in] Binary reduction functor
-        int                         num_items,                      ///< [in] Total number of associated key+value pairs (i.e., the length of \p d_in_keys and \p d_in_values)
-        cudaStream_t                stream             = 0,         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous  = false)     ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        // FlagT iterator type (not used)
-
-        // Selection op (not used)
-
-        // Default == operator
-        typedef Equality EqualityOp;
-
-        return DispatchReduceByKey<KeysInputIteratorT, UniqueOutputIteratorT, ValuesInputIteratorT, AggregatesOutputIteratorT, NumRunsOutputIteratorT, EqualityOp, ReductionOpT, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_keys_in,
-            d_unique_out,
-            d_values_in,
-            d_aggregates_out,
-            d_num_runs_out,
-            EqualityOp(),
-            reduction_op,
-            num_items,
-            stream,
-            debug_synchronous);
-    }
-
-};
-
-/**
- * \example example_device_reduce.cu
- */
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/device/device_run_length_encode.cuh b/pfsimulator/third_party/cub/device/device_run_length_encode.cuh
deleted file mode 100644
index 7a2e82d9d..000000000
--- a/pfsimulator/third_party/cub/device/device_run_length_encode.cuh
+++ /dev/null
@@ -1,278 +0,0 @@
-
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::DeviceRunLengthEncode provides device-wide, parallel operations for computing a run-length encoding across a sequence of data items residing within device-accessible memory.
- */
-
-#pragma once
-
-#include <stdio.h>
-#include <iterator>
-
-#include "dispatch/dispatch_rle.cuh"
-#include "dispatch/dispatch_reduce_by_key.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \brief DeviceRunLengthEncode provides device-wide, parallel operations for demarcating "runs" of same-valued items within a sequence residing within device-accessible memory. ![](run_length_encode_logo.png)
- * \ingroup SingleModule
- *
- * \par Overview
- * A <a href="http://en.wikipedia.org/wiki/Run-length_encoding"><em>run-length encoding</em></a>
- * computes a simple compressed representation of a sequence of input elements such that each
- * maximal "run" of consecutive same-valued data items is encoded as a single data value along with a
- * count of the elements in that run.
- *
- * \par Usage Considerations
- * \cdp_class{DeviceRunLengthEncode}
- *
- * \par Performance
- * \linear_performance{run-length encode}
- *
- * \par
- * The following chart illustrates DeviceRunLengthEncode::RunLengthEncode performance across
- * different CUDA architectures for \p int32 items.
- * Segments have lengths uniformly sampled from [1,1000].
- *
- * \image html rle_int32_len_500.png
- *
- * \par
- * \plots_below
- *
- */
-struct DeviceRunLengthEncode
-{
-
-    /**
-     * \brief Computes a run-length encoding of the sequence \p d_in.
-     *
-     * \par
-     * - For the <em>i</em><sup>th</sup> run encountered, the first key of the run and its length are written to
-     *   <tt>d_unique_out[<em>i</em>]</tt> and <tt>d_counts_out[<em>i</em>]</tt>,
-     *   respectively.
-     * - The total number of runs encountered is written to \p d_num_runs_out.
-     * - The <tt>==</tt> equality operator is used to determine whether values are equivalent
-     * - \devicestorage
-     *
-     * \par Performance
-     * The following charts illustrate saturated encode performance across different
-     * CUDA architectures for \p int32 and \p int64 items, respectively.  Segments have
-     * lengths uniformly sampled from [1,1000].
-     *
-     * \image html rle_int32_len_500.png
-     * \image html rle_int64_len_500.png
-     *
-     * \par
-     * The following charts are similar, but with segment lengths uniformly sampled from [1,10]:
-     *
-     * \image html rle_int32_len_5.png
-     * \image html rle_int64_len_5.png
-     *
-     * \par Snippet
-     * The code snippet below illustrates the run-length encoding of a sequence of \p int values.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_run_length_encode.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int          num_items;          // e.g., 8
-     * int          *d_in;              // e.g., [0, 2, 2, 9, 5, 5, 5, 8]
-     * int          *d_unique_out;      // e.g., [ ,  ,  ,  ,  ,  ,  ,  ]
-     * int          *d_counts_out;      // e.g., [ ,  ,  ,  ,  ,  ,  ,  ]
-     * int          *d_num_runs_out;    // e.g., [ ]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceRunLengthEncode::Encode(d_temp_storage, temp_storage_bytes, d_in, d_unique_out, d_counts_out, d_num_runs_out, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run encoding
-     * cub::DeviceRunLengthEncode::Encode(d_temp_storage, temp_storage_bytes, d_in, d_unique_out, d_counts_out, d_num_runs_out, num_items);
-     *
-     * // d_unique_out      <-- [0, 2, 9, 5, 8]
-     * // d_counts_out      <-- [1, 2, 1, 3, 1]
-     * // d_num_runs_out    <-- [5]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT           <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
-     * \tparam UniqueOutputIteratorT    <b>[inferred]</b> Random-access output iterator type for writing unique output items \iterator
-     * \tparam LengthsOutputIteratorT   <b>[inferred]</b> Random-access output iterator type for writing output counts \iterator
-     * \tparam NumRunsOutputIteratorT   <b>[inferred]</b> Output iterator type for recording the number of runs encountered \iterator
-     */
-    template <
-        typename                    InputIteratorT,
-        typename                    UniqueOutputIteratorT,
-        typename                    LengthsOutputIteratorT,
-        typename                    NumRunsOutputIteratorT>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Encode(
-        void*                       d_temp_storage,                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                      &temp_storage_bytes,            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT              d_in,                           ///< [in] Pointer to the input sequence of keys
-        UniqueOutputIteratorT       d_unique_out,                   ///< [out] Pointer to the output sequence of unique keys (one key per run)
-        LengthsOutputIteratorT      d_counts_out,                   ///< [out] Pointer to the output sequence of run-lengths (one count per run)
-        NumRunsOutputIteratorT      d_num_runs_out,                     ///< [out] Pointer to total number of runs
-        int                         num_items,                      ///< [in] Total number of associated key+value pairs (i.e., the length of \p d_in_keys and \p d_in_values)
-        cudaStream_t                stream             = 0,         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous  = false)     ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        typedef int         OffsetT;                    // Signed integer type for global offsets
-        typedef NullType*   FlagIterator;               // FlagT iterator type (not used)
-        typedef NullType    SelectOp;                   // Selection op (not used)
-        typedef Equality    EqualityOp;                 // Default == operator
-        typedef cub::Sum    ReductionOp;                // Value reduction operator
-
-        // The lengths output value type
-        typedef typename If<(Equals<typename std::iterator_traits<LengthsOutputIteratorT>::value_type, void>::VALUE),   // LengthT =  (if output iterator's value type is void) ?
-            OffsetT,                                                                                                    // ... then the OffsetT type,
-            typename std::iterator_traits<LengthsOutputIteratorT>::value_type>::Type LengthT;                           // ... else the output iterator's value type
-
-        // Generator type for providing 1s values for run-length reduction
-        typedef ConstantInputIterator<LengthT, OffsetT> LengthsInputIteratorT;
-
-        return DispatchReduceByKey<InputIteratorT, UniqueOutputIteratorT, LengthsInputIteratorT, LengthsOutputIteratorT, NumRunsOutputIteratorT, EqualityOp, ReductionOp, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            d_unique_out,
-            LengthsInputIteratorT((LengthT) 1),
-            d_counts_out,
-            d_num_runs_out,
-            EqualityOp(),
-            ReductionOp(),
-            num_items,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Enumerates the starting offsets and lengths of all non-trivial runs (of length > 1) of same-valued keys in the sequence \p d_in.
-     *
-     * \par
-     * - For the <em>i</em><sup>th</sup> non-trivial run, the run's starting offset
-     *   and its length are written to <tt>d_offsets_out[<em>i</em>]</tt> and
-     *   <tt>d_lengths_out[<em>i</em>]</tt>, respectively.
-     * - The total number of runs encountered is written to \p d_num_runs_out.
-     * - The <tt>==</tt> equality operator is used to determine whether values are equivalent
-     * - \devicestorage
-     *
-     * \par Performance
-     *
-     * \par Snippet
-     * The code snippet below illustrates the identification of non-trivial runs within a sequence of \p int values.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_run_length_encode.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int          num_items;          // e.g., 8
-     * int          *d_in;              // e.g., [0, 2, 2, 9, 5, 5, 5, 8]
-     * int          *d_offsets_out;     // e.g., [ ,  ,  ,  ,  ,  ,  ,  ]
-     * int          *d_lengths_out;     // e.g., [ ,  ,  ,  ,  ,  ,  ,  ]
-     * int          *d_num_runs_out;    // e.g., [ ]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceRunLengthEncode::NonTrivialRuns(d_temp_storage, temp_storage_bytes, d_in, d_offsets_out, d_lengths_out, d_num_runs_out, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run encoding
-     * cub::DeviceRunLengthEncode::NonTrivialRuns(d_temp_storage, temp_storage_bytes, d_in, d_offsets_out, d_lengths_out, d_num_runs_out, num_items);
-     *
-     * // d_offsets_out         <-- [1, 4]
-     * // d_lengths_out         <-- [2, 3]
-     * // d_num_runs_out        <-- [2]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT           <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
-     * \tparam OffsetsOutputIteratorT   <b>[inferred]</b> Random-access output iterator type for writing run-offset values \iterator
-     * \tparam LengthsOutputIteratorT   <b>[inferred]</b> Random-access output iterator type for writing run-length values \iterator
-     * \tparam NumRunsOutputIteratorT   <b>[inferred]</b> Output iterator type for recording the number of runs encountered \iterator
-     */
-    template <
-        typename                InputIteratorT,
-        typename                OffsetsOutputIteratorT,
-        typename                LengthsOutputIteratorT,
-        typename                NumRunsOutputIteratorT>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t NonTrivialRuns(
-        void*               d_temp_storage,                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                  &temp_storage_bytes,            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT          d_in,                           ///< [in] Pointer to input sequence of data items
-        OffsetsOutputIteratorT  d_offsets_out,                  ///< [out] Pointer to output sequence of run-offsets (one offset per non-trivial run)
-        LengthsOutputIteratorT  d_lengths_out,                  ///< [out] Pointer to output sequence of run-lengths (one count per non-trivial run)
-        NumRunsOutputIteratorT  d_num_runs_out,                 ///< [out] Pointer to total number of runs (i.e., length of \p d_offsets_out)
-        int                     num_items,                      ///< [in] Total number of associated key+value pairs (i.e., the length of \p d_in_keys and \p d_in_values)
-        cudaStream_t            stream             = 0,         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                    debug_synchronous  = false)     ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        typedef int         OffsetT;                    // Signed integer type for global offsets
-        typedef Equality    EqualityOp;                 // Default == operator
-
-        return DeviceRleDispatch<InputIteratorT, OffsetsOutputIteratorT, LengthsOutputIteratorT, NumRunsOutputIteratorT, EqualityOp, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            d_offsets_out,
-            d_lengths_out,
-            d_num_runs_out,
-            EqualityOp(),
-            num_items,
-            stream,
-            debug_synchronous);
-    }
-
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/device/device_scan.cuh b/pfsimulator/third_party/cub/device/device_scan.cuh
deleted file mode 100644
index e86fefe3c..000000000
--- a/pfsimulator/third_party/cub/device/device_scan.cuh
+++ /dev/null
@@ -1,443 +0,0 @@
-
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::DeviceScan provides device-wide, parallel operations for computing a prefix scan across a sequence of data items residing within device-accessible memory.
- */
-
-#pragma once
-
-#include <stdio.h>
-#include <iterator>
-
-#include "dispatch/dispatch_scan.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \brief DeviceScan provides device-wide, parallel operations for computing a prefix scan across a sequence of data items residing within device-accessible memory. ![](device_scan.png)
- * \ingroup SingleModule
- *
- * \par Overview
- * Given a sequence of input elements and a binary reduction operator, a [<em>prefix scan</em>](http://en.wikipedia.org/wiki/Prefix_sum)
- * produces an output sequence where each element is computed to be the reduction
- * of the elements occurring earlier in the input sequence.  <em>Prefix sum</em>
- * connotes a prefix scan with the addition operator. The term \em inclusive indicates
- * that the <em>i</em><sup>th</sup> output reduction incorporates the <em>i</em><sup>th</sup> input.
- * The term \em exclusive indicates the <em>i</em><sup>th</sup> input is not incorporated into
- * the <em>i</em><sup>th</sup> output reduction.
- *
- * \par
- * As of CUB 1.0.1 (2013), CUB's device-wide scan APIs have implemented our <em>"decoupled look-back"</em> algorithm
- * for performing global prefix scan with only a single pass through the
- * input data, as described in our 2016 technical report [1].  The central
- * idea is to leverage a small, constant factor of redundant work in order to overlap the latencies
- * of global prefix propagation with local computation.  As such, our algorithm requires only
- * ~2<em>n</em> data movement (<em>n</em> inputs are read, <em>n</em> outputs are written), and typically
- * proceeds at "memcpy" speeds.
- *
- * \par
- * [1] [Duane Merrill and Michael Garland.  "Single-pass Parallel Prefix Scan with Decoupled Look-back", <em>NVIDIA Technical Report NVR-2016-002</em>, 2016.](https://research.nvidia.com/publication/single-pass-parallel-prefix-scan-decoupled-look-back)
- *
- * \par Usage Considerations
- * \cdp_class{DeviceScan}
- *
- * \par Performance
- * \linear_performance{prefix scan}
- *
- * \par
- * The following chart illustrates DeviceScan::ExclusiveSum
- * performance across different CUDA architectures for \p int32 keys.
- * \plots_below
- *
- * \image html scan_int32.png
- *
- */
-struct DeviceScan
-{
-    /******************************************************************//**
-     * \name Exclusive scans
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Computes a device-wide exclusive prefix sum.  The value of 0 is applied as the initial value, and is assigned to *d_out.
-     *
-     * \par
-     * - Supports non-commutative sum operators.
-     * - Provides "run-to-run" determinism for pseudo-associative reduction
-     *   (e.g., addition of floating point types) on the same GPU device.
-     *   However, results for pseudo-associative reduction may be inconsistent
-     *   from one device to a another device of a different compute-capability
-     *   because CUB can employ different tile-sizing for different architectures.
-     * - \devicestorage
-     *
-     * \par Performance
-     * The following charts illustrate saturated exclusive sum performance across different
-     * CUDA architectures for \p int32 and \p int64 items, respectively.
-     *
-     * \image html scan_int32.png
-     * \image html scan_int64.png
-     *
-     * \par Snippet
-     * The code snippet below illustrates the exclusive prefix sum of an \p int device vector.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_scan.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int  num_items;      // e.g., 7
-     * int  *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_out;         // e.g., [ ,  ,  ,  ,  ,  ,  ]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceScan::ExclusiveSum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run exclusive prefix sum
-     * cub::DeviceScan::ExclusiveSum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items);
-     *
-     * // d_out s<-- [0, 8, 14, 21, 26, 29, 29]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading scan inputs \iterator
-     * \tparam OutputIteratorT    <b>[inferred]</b> Random-access output iterator type for writing scan outputs \iterator
-     */
-    template <
-        typename        InputIteratorT,
-        typename        OutputIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t ExclusiveSum(
-        void            *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t          &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT  d_in,                               ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT d_out,                              ///< [out] Pointer to the output sequence of data items
-        int             num_items,                          ///< [in] Total number of input items (i.e., the length of \p d_in)
-        cudaStream_t    stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool            debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        // The output value type
-        typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
-            typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
-            typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
-
-        // Initial value
-        OutputT init_value = 0;
-
-        return DispatchScan<InputIteratorT, OutputIteratorT, Sum, OutputT, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            d_out,
-            Sum(),
-            init_value,
-            num_items,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Computes a device-wide exclusive prefix scan using the specified binary \p scan_op functor.  The \p init_value value is applied as the initial value, and is assigned to *d_out.
-     *
-     * \par
-     * - Supports non-commutative scan operators.
-     * - Provides "run-to-run" determinism for pseudo-associative reduction
-     *   (e.g., addition of floating point types) on the same GPU device.
-     *   However, results for pseudo-associative reduction may be inconsistent
-     *   from one device to a another device of a different compute-capability
-     *   because CUB can employ different tile-sizing for different architectures.
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the exclusive prefix min-scan of an \p int device vector
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_scan.cuh>
-     *
-     * // CustomMin functor
-     * struct CustomMin
-     * {
-     *     template <typename T>
-     *     CUB_RUNTIME_FUNCTION __forceinline__
-     *     T operator()(const T &a, const T &b) const {
-     *         return (b < a) ? b : a;
-     *     }
-     * };
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int          num_items;      // e.g., 7
-     * int          *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int          *d_out;         // e.g., [ ,  ,  ,  ,  ,  ,  ]
-     * CustomMin    min_op
-     * ...
-     *
-     * // Determine temporary device storage requirements for exclusive prefix scan
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceScan::ExclusiveScan(d_temp_storage, temp_storage_bytes, d_in, d_out, min_op, (int) MAX_INT, num_items);
-     *
-     * // Allocate temporary storage for exclusive prefix scan
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run exclusive prefix min-scan
-     * cub::DeviceScan::ExclusiveScan(d_temp_storage, temp_storage_bytes, d_in, d_out, min_op, (int) MAX_INT, num_items);
-     *
-     * // d_out <-- [2147483647, 8, 6, 6, 5, 3, 0]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT   <b>[inferred]</b> Random-access input iterator type for reading scan inputs \iterator
-     * \tparam OutputIteratorT  <b>[inferred]</b> Random-access output iterator type for writing scan outputs \iterator
-     * \tparam ScanOp           <b>[inferred]</b> Binary scan functor type having member <tt>T operator()(const T &a, const T &b)</tt>
-     * \tparam Identity         <b>[inferred]</b> Type of the \p identity value used Binary scan functor type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <
-        typename        InputIteratorT,
-        typename        OutputIteratorT,
-        typename        ScanOpT,
-        typename        InitValueT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t ExclusiveScan(
-        void            *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t          &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT  d_in,                               ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT d_out,                              ///< [out] Pointer to the output sequence of data items
-        ScanOpT         scan_op,                            ///< [in] Binary scan functor
-        InitValueT      init_value,                         ///< [in] Initial value to seed the exclusive scan (and is assigned to *d_out)
-        int             num_items,                          ///< [in] Total number of input items (i.e., the length of \p d_in)
-        cudaStream_t    stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool            debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        return DispatchScan<InputIteratorT, OutputIteratorT, ScanOpT, InitValueT, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            d_out,
-            scan_op,
-            init_value,
-            num_items,
-            stream,
-            debug_synchronous);
-    }
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Inclusive scans
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Computes a device-wide inclusive prefix sum.
-     *
-     * \par
-     * - Supports non-commutative sum operators.
-     * - Provides "run-to-run" determinism for pseudo-associative reduction
-     *   (e.g., addition of floating point types) on the same GPU device.
-     *   However, results for pseudo-associative reduction may be inconsistent
-     *   from one device to a another device of a different compute-capability
-     *   because CUB can employ different tile-sizing for different architectures.
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the inclusive prefix sum of an \p int device vector.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_scan.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int  num_items;      // e.g., 7
-     * int  *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_out;         // e.g., [ ,  ,  ,  ,  ,  ,  ]
-     * ...
-     *
-     * // Determine temporary device storage requirements for inclusive prefix sum
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceScan::InclusiveSum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items);
-     *
-     * // Allocate temporary storage for inclusive prefix sum
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run inclusive prefix sum
-     * cub::DeviceScan::InclusiveSum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items);
-     *
-     * // d_out <-- [8, 14, 21, 26, 29, 29, 38]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading scan inputs \iterator
-     * \tparam OutputIteratorT    <b>[inferred]</b> Random-access output iterator type for writing scan outputs \iterator
-     */
-    template <
-        typename            InputIteratorT,
-        typename            OutputIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t InclusiveSum(
-        void*               d_temp_storage,                 ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&             temp_storage_bytes,             ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT      d_in,                           ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT     d_out,                          ///< [out] Pointer to the output sequence of data items
-        int                 num_items,                      ///< [in] Total number of input items (i.e., the length of \p d_in)
-        cudaStream_t        stream             = 0,         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous  = false)     ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        return DispatchScan<InputIteratorT, OutputIteratorT, Sum, NullType, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            d_out,
-            Sum(),
-            NullType(),
-            num_items,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Computes a device-wide inclusive prefix scan using the specified binary \p scan_op functor.
-     *
-     * \par
-     * - Supports non-commutative scan operators.
-     * - Provides "run-to-run" determinism for pseudo-associative reduction
-     *   (e.g., addition of floating point types) on the same GPU device.
-     *   However, results for pseudo-associative reduction may be inconsistent
-     *   from one device to a another device of a different compute-capability
-     *   because CUB can employ different tile-sizing for different architectures.
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the inclusive prefix min-scan of an \p int device vector.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_scan.cuh>
-     *
-     * // CustomMin functor
-     * struct CustomMin
-     * {
-     *     template <typename T>
-     *     CUB_RUNTIME_FUNCTION __forceinline__
-     *     T operator()(const T &a, const T &b) const {
-     *         return (b < a) ? b : a;
-     *     }
-     * };
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int          num_items;      // e.g., 7
-     * int          *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int          *d_out;         // e.g., [ ,  ,  ,  ,  ,  ,  ]
-     * CustomMin    min_op;
-     * ...
-     *
-     * // Determine temporary device storage requirements for inclusive prefix scan
-     * void *d_temp_storage = NULL;
-     * size_t temp_storage_bytes = 0;
-     * cub::DeviceScan::InclusiveScan(d_temp_storage, temp_storage_bytes, d_in, d_out, min_op, num_items);
-     *
-     * // Allocate temporary storage for inclusive prefix scan
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run inclusive prefix min-scan
-     * cub::DeviceScan::InclusiveScan(d_temp_storage, temp_storage_bytes, d_in, d_out, min_op, num_items);
-     *
-     * // d_out <-- [8, 6, 6, 5, 3, 0, 0]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT   <b>[inferred]</b> Random-access input iterator type for reading scan inputs \iterator
-     * \tparam OutputIteratorT  <b>[inferred]</b> Random-access output iterator type for writing scan outputs \iterator
-     * \tparam ScanOp           <b>[inferred]</b> Binary scan functor type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <
-        typename        InputIteratorT,
-        typename        OutputIteratorT,
-        typename        ScanOpT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t InclusiveScan(
-        void            *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t          &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT  d_in,                               ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT d_out,                              ///< [out] Pointer to the output sequence of data items
-        ScanOpT         scan_op,                            ///< [in] Binary scan functor
-        int             num_items,                          ///< [in] Total number of input items (i.e., the length of \p d_in)
-        cudaStream_t    stream             = 0,             ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool            debug_synchronous  = false)         ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        return DispatchScan<InputIteratorT, OutputIteratorT, ScanOpT, NullType, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            d_out,
-            scan_op,
-            NullType(),
-            num_items,
-            stream,
-            debug_synchronous);
-    }
-
-    //@}  end member group
-
-};
-
-/**
- * \example example_device_scan.cu
- */
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/device/device_segmented_radix_sort.cuh b/pfsimulator/third_party/cub/device/device_segmented_radix_sort.cuh
deleted file mode 100644
index 0d3607627..000000000
--- a/pfsimulator/third_party/cub/device/device_segmented_radix_sort.cuh
+++ /dev/null
@@ -1,876 +0,0 @@
-
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::DeviceSegmentedRadixSort provides device-wide, parallel operations for computing a batched radix sort across multiple, non-overlapping sequences of data items residing within device-accessible memory.
- */
-
-#pragma once
-
-#include <stdio.h>
-#include <iterator>
-
-#include "dispatch/dispatch_radix_sort.cuh"
-#include "../util_arch.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \brief DeviceSegmentedRadixSort provides device-wide, parallel operations for computing a batched radix sort across multiple, non-overlapping sequences of data items residing within device-accessible memory. ![](segmented_sorting_logo.png)
- * \ingroup SegmentedModule
- *
- * \par Overview
- * The [<em>radix sorting method</em>](http://en.wikipedia.org/wiki/Radix_sort) arranges
- * items into ascending (or descending) order.  The algorithm relies upon a positional representation for
- * keys, i.e., each key is comprised of an ordered sequence of symbols (e.g., digits,
- * characters, etc.) specified from least-significant to most-significant.  For a
- * given input sequence of keys and a set of rules specifying a total ordering
- * of the symbolic alphabet, the radix sorting method produces a lexicographic
- * ordering of those keys.
- *
- * \par
- * DeviceSegmentedRadixSort can sort all of the built-in C++ numeric primitive types
- * (<tt>unsigned char</tt>, \p int, \p double, etc.) as well as CUDA's \p __half
- * half-precision floating-point type.  Although the direct radix sorting
- * method can only be applied to unsigned integral types, DeviceSegmentedRadixSort
- * is able to sort signed and floating-point types via simple bit-wise transformations
- * that ensure lexicographic key ordering.
- *
- * \par Usage Considerations
- * \cdp_class{DeviceSegmentedRadixSort}
- *
- */
-struct DeviceSegmentedRadixSort
-{
-
-    /******************************************************************//**
-     * \name Key-value pairs
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Sorts segments of key-value pairs into ascending order. (~<em>2N </em>auxiliary storage required)
-     *
-     * \par
-     * - The contents of the input data are not altered by the sorting operation
-     * - When input a contiguous sequence of segments, a single sequence
-     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
-     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
-     *   the latter is specified as <tt>segment_offsets+1</tt>).
-     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
-     * - \devicestorageNP  For sorting using only <em>O</em>(<tt>P</tt>) temporary storage, see the sorting interface using DoubleBuffer wrappers below.
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the batched sorting of three segments (with one zero-length segment) of \p int keys
-     * with associated vector of \p int values.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_segmentd_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for sorting data
-     * int  num_items;          // e.g., 7
-     * int  num_segments;       // e.g., 3
-     * int  *d_offsets;         // e.g., [0, 3, 3, 7]
-     * int  *d_keys_in;         // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_keys_out;        // e.g., [-, -, -, -, -, -, -]
-     * int  *d_values_in;       // e.g., [0, 1, 2, 3, 4, 5, 6]
-     * int  *d_values_out;      // e.g., [-, -, -, -, -, -, -]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSegmentedRadixSort::SortPairs(d_temp_storage, temp_storage_bytes,
-     *     d_keys_in, d_keys_out, d_values_in, d_values_out,
-     *     num_items, num_segments, d_offsets, d_offsets + 1);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run sorting operation
-     * cub::DeviceSegmentedRadixSort::SortPairs(d_temp_storage, temp_storage_bytes,
-     *     d_keys_in, d_keys_out, d_values_in, d_values_out,
-     *     num_items, num_segments, d_offsets, d_offsets + 1);
-     *
-     * // d_keys_out            <-- [6, 7, 8, 0, 3, 5, 9]
-     * // d_values_out          <-- [1, 2, 0, 5, 4, 3, 6]
-     *
-     * \endcode
-     *
-     * \tparam KeyT             <b>[inferred]</b> Key type
-     * \tparam ValueT           <b>[inferred]</b> Value type
-     * \tparam OffsetIteratorT  <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
-     */
-    template <
-        typename            KeyT,
-        typename            ValueT,
-        typename            OffsetIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t SortPairs(
-        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        const KeyT          *d_keys_in,                             ///< [in] %Device-accessible pointer to the input data of key data to sort
-        KeyT                *d_keys_out,                            ///< [out] %Device-accessible pointer to the sorted output sequence of key data
-        const ValueT        *d_values_in,                           ///< [in] %Device-accessible pointer to the corresponding input sequence of associated value items
-        ValueT              *d_values_out,                          ///< [out] %Device-accessible pointer to the correspondingly-reordered output sequence of associated value items
-        int                 num_items,                              ///< [in] The total number of items to sort (across all segments)
-        int                 num_segments,                           ///< [in] The number of segments that comprise the sorting data
-        OffsetIteratorT     d_begin_offsets,                        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-        OffsetIteratorT     d_end_offsets,                          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
-        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
-        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        DoubleBuffer<KeyT>       d_keys(const_cast<KeyT*>(d_keys_in), d_keys_out);
-        DoubleBuffer<ValueT>     d_values(const_cast<ValueT*>(d_values_in), d_values_out);
-
-        return DispatchSegmentedRadixSort<false, KeyT, ValueT, OffsetIteratorT, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_keys,
-            d_values,
-            num_items,
-            num_segments,
-            d_begin_offsets,
-            d_end_offsets,
-            begin_bit,
-            end_bit,
-            false,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Sorts segments of key-value pairs into ascending order. (~<em>N </em>auxiliary storage required)
-     *
-     * \par
-     * - The sorting operation is given a pair of key buffers and a corresponding
-     *   pair of associated value buffers.  Each pair is managed by a DoubleBuffer
-     *   structure that indicates which of the two buffers is "current" (and thus
-     *   contains the input data to be sorted).
-     * - The contents of both buffers within each pair may be altered by the sorting
-     *   operation.
-     * - Upon completion, the sorting operation will update the "current" indicator
-     *   within each DoubleBuffer wrapper to reference which of the two buffers
-     *   now contains the sorted output sequence (a function of the number of key bits
-     *   specified and the targeted device architecture).
-     * - When input a contiguous sequence of segments, a single sequence
-     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
-     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
-     *   the latter is specified as <tt>segment_offsets+1</tt>).
-     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
-     * - \devicestorageP
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the batched sorting of three segments (with one zero-length segment) of \p int keys
-     * with associated vector of \p int values.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_segmentd_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for sorting data
-     * int  num_items;          // e.g., 7
-     * int  num_segments;       // e.g., 3
-     * int  *d_offsets;         // e.g., [0, 3, 3, 7]
-     * int  *d_key_buf;         // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_key_alt_buf;     // e.g., [-, -, -, -, -, -, -]
-     * int  *d_value_buf;       // e.g., [0, 1, 2, 3, 4, 5, 6]
-     * int  *d_value_alt_buf;   // e.g., [-, -, -, -, -, -, -]
-     * ...
-     *
-     * // Create a set of DoubleBuffers to wrap pairs of device pointers
-     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
-     * cub::DoubleBuffer<int> d_values(d_value_buf, d_value_alt_buf);
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSegmentedRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, d_keys, d_values,
-     *     num_items, num_segments, d_offsets, d_offsets + 1);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run sorting operation
-     * cub::DeviceSegmentedRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, d_keys, d_values,
-     *     num_items, num_segments, d_offsets, d_offsets + 1);
-     *
-     * // d_keys.Current()      <-- [6, 7, 8, 0, 3, 5, 9]
-     * // d_values.Current()    <-- [5, 4, 3, 1, 2, 0, 6]
-     *
-     * \endcode
-     *
-     * \tparam KeyT             <b>[inferred]</b> Key type
-     * \tparam ValueT           <b>[inferred]</b> Value type
-     * \tparam OffsetIteratorT  <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
-     */
-    template <
-        typename                KeyT,
-        typename                ValueT,
-        typename                OffsetIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t SortPairs(
-        void                    *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                  &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        DoubleBuffer<KeyT>      &d_keys,                                ///< [in,out] Reference to the double-buffer of keys whose "current" device-accessible buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
-        DoubleBuffer<ValueT>    &d_values,                              ///< [in,out] Double-buffer of values whose "current" device-accessible buffer contains the unsorted input values and, upon return, is updated to point to the sorted output values
-        int                     num_items,                              ///< [in] The total number of items to sort (across all segments)
-        int                     num_segments,                           ///< [in] The number of segments that comprise the sorting data
-        OffsetIteratorT         d_begin_offsets,                        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-        OffsetIteratorT         d_end_offsets,                          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-        int                     begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
-        int                     end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
-        cudaStream_t            stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                    debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        return DispatchSegmentedRadixSort<false, KeyT, ValueT, OffsetIteratorT, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_keys,
-            d_values,
-            num_items,
-            num_segments,
-            d_begin_offsets,
-            d_end_offsets,
-            begin_bit,
-            end_bit,
-            true,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Sorts segments of key-value pairs into descending order. (~<em>2N</em> auxiliary storage required).
-     *
-     * \par
-     * - The contents of the input data are not altered by the sorting operation
-     * - When input a contiguous sequence of segments, a single sequence
-     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
-     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
-     *   the latter is specified as <tt>segment_offsets+1</tt>).
-     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
-     * - \devicestorageNP  For sorting using only <em>O</em>(<tt>P</tt>) temporary storage, see the sorting interface using DoubleBuffer wrappers below.
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the batched sorting of three segments (with one zero-length segment) of \p int keys
-     * with associated vector of \p int values.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_segmentd_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for sorting data
-     * int  num_items;          // e.g., 7
-     * int  num_segments;       // e.g., 3
-     * int  *d_offsets;         // e.g., [0, 3, 3, 7]
-     * int  *d_keys_in;         // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_keys_out;        // e.g., [-, -, -, -, -, -, -]
-     * int  *d_values_in;       // e.g., [0, 1, 2, 3, 4, 5, 6]
-     * int  *d_values_out;      // e.g., [-, -, -, -, -, -, -]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSegmentedRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes,
-     *     d_keys_in, d_keys_out, d_values_in, d_values_out,
-     *     num_items, num_segments, d_offsets, d_offsets + 1);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run sorting operation
-     * cub::DeviceSegmentedRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes,
-     *     d_keys_in, d_keys_out, d_values_in, d_values_out,
-     *     num_items, num_segments, d_offsets, d_offsets + 1);
-     *
-     * // d_keys_out            <-- [8, 7, 6, 9, 5, 3, 0]
-     * // d_values_out          <-- [0, 2, 1, 6, 3, 4, 5]
-     *
-     * \endcode
-     *
-     * \tparam KeyT             <b>[inferred]</b> Key type
-     * \tparam ValueT           <b>[inferred]</b> Value type
-     * \tparam OffsetIteratorT  <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
-     */
-    template <
-        typename            KeyT,
-        typename            ValueT,
-        typename            OffsetIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t SortPairsDescending(
-        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        const KeyT          *d_keys_in,                             ///< [in] %Device-accessible pointer to the input data of key data to sort
-        KeyT                *d_keys_out,                            ///< [out] %Device-accessible pointer to the sorted output sequence of key data
-        const ValueT        *d_values_in,                           ///< [in] %Device-accessible pointer to the corresponding input sequence of associated value items
-        ValueT              *d_values_out,                          ///< [out] %Device-accessible pointer to the correspondingly-reordered output sequence of associated value items
-        int                 num_items,                              ///< [in] The total number of items to sort (across all segments)
-        int                 num_segments,                           ///< [in] The number of segments that comprise the sorting data
-        OffsetIteratorT     d_begin_offsets,                        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-        OffsetIteratorT     d_end_offsets,                          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
-        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
-        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        DoubleBuffer<KeyT>       d_keys(const_cast<KeyT*>(d_keys_in), d_keys_out);
-        DoubleBuffer<ValueT>     d_values(const_cast<ValueT*>(d_values_in), d_values_out);
-
-        return DispatchSegmentedRadixSort<true, KeyT, ValueT, OffsetIteratorT, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_keys,
-            d_values,
-            num_items,
-            num_segments,
-            d_begin_offsets,
-            d_end_offsets,
-            begin_bit,
-            end_bit,
-            false,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Sorts segments of key-value pairs into descending order. (~<em>N </em>auxiliary storage required).
-     *
-     * \par
-     * - The sorting operation is given a pair of key buffers and a corresponding
-     *   pair of associated value buffers.  Each pair is managed by a DoubleBuffer
-     *   structure that indicates which of the two buffers is "current" (and thus
-     *   contains the input data to be sorted).
-     * - The contents of both buffers within each pair may be altered by the sorting
-     *   operation.
-     * - Upon completion, the sorting operation will update the "current" indicator
-     *   within each DoubleBuffer wrapper to reference which of the two buffers
-     *   now contains the sorted output sequence (a function of the number of key bits
-     *   specified and the targeted device architecture).
-     * - When input a contiguous sequence of segments, a single sequence
-     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
-     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
-     *   the latter is specified as <tt>segment_offsets+1</tt>).
-     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
-     * - \devicestorageP
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the batched sorting of three segments (with one zero-length segment) of \p int keys
-     * with associated vector of \p int values.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_segmentd_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for sorting data
-     * int  num_items;          // e.g., 7
-     * int  num_segments;       // e.g., 3
-     * int  *d_offsets;         // e.g., [0, 3, 3, 7]
-     * int  *d_key_buf;         // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_key_alt_buf;     // e.g., [-, -, -, -, -, -, -]
-     * int  *d_value_buf;       // e.g., [0, 1, 2, 3, 4, 5, 6]
-     * int  *d_value_alt_buf;   // e.g., [-, -, -, -, -, -, -]
-     * ...
-     *
-     * // Create a set of DoubleBuffers to wrap pairs of device pointers
-     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
-     * cub::DoubleBuffer<int> d_values(d_value_buf, d_value_alt_buf);
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSegmentedRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes, d_keys, d_values,
-     *     num_items, num_segments, d_offsets, d_offsets + 1);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run sorting operation
-     * cub::DeviceSegmentedRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes, d_keys, d_values,
-     *     num_items, num_segments, d_offsets, d_offsets + 1);
-     *
-     * // d_keys.Current()      <-- [8, 7, 6, 9, 5, 3, 0]
-     * // d_values.Current()    <-- [0, 2, 1, 6, 3, 4, 5]
-     *
-     * \endcode
-     *
-     * \tparam KeyT             <b>[inferred]</b> Key type
-     * \tparam ValueT           <b>[inferred]</b> Value type
-     * \tparam OffsetIteratorT  <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
-     */
-    template <
-        typename                KeyT,
-        typename                ValueT,
-        typename                OffsetIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t SortPairsDescending(
-        void                    *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                  &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        DoubleBuffer<KeyT>      &d_keys,                                ///< [in,out] Reference to the double-buffer of keys whose "current" device-accessible buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
-        DoubleBuffer<ValueT>    &d_values,                              ///< [in,out] Double-buffer of values whose "current" device-accessible buffer contains the unsorted input values and, upon return, is updated to point to the sorted output values
-        int                     num_items,                              ///< [in] The total number of items to sort (across all segments)
-        int                     num_segments,                           ///< [in] The number of segments that comprise the sorting data
-        OffsetIteratorT         d_begin_offsets,                        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-        OffsetIteratorT         d_end_offsets,                          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-        int                     begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
-        int                     end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
-        cudaStream_t            stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                    debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        return DispatchSegmentedRadixSort<true, KeyT, ValueT, OffsetIteratorT, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_keys,
-            d_values,
-            num_items,
-            num_segments,
-            d_begin_offsets,
-            d_end_offsets,
-            begin_bit,
-            end_bit,
-            true,
-            stream,
-            debug_synchronous);
-    }
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Keys-only
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Sorts segments of keys into ascending order. (~<em>2N </em>auxiliary storage required)
-     *
-     * \par
-     * - The contents of the input data are not altered by the sorting operation
-     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
-     * - When input a contiguous sequence of segments, a single sequence
-     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
-     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
-     *   the latter is specified as <tt>segment_offsets+1</tt>).
-     * - \devicestorageNP  For sorting using only <em>O</em>(<tt>P</tt>) temporary storage, see the sorting interface using DoubleBuffer wrappers below.
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the batched sorting of three segments (with one zero-length segment) of \p int keys.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_segmentd_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for sorting data
-     * int  num_items;          // e.g., 7
-     * int  num_segments;       // e.g., 3
-     * int  *d_offsets;         // e.g., [0, 3, 3, 7]
-     * int  *d_keys_in;         // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_keys_out;        // e.g., [-, -, -, -, -, -, -]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSegmentedRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out,
-     *     num_items, num_segments, d_offsets, d_offsets + 1);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run sorting operation
-     * cub::DeviceSegmentedRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out,
-     *     num_items, num_segments, d_offsets, d_offsets + 1);
-     *
-     * // d_keys_out            <-- [6, 7, 8, 0, 3, 5, 9]
-     *
-     * \endcode
-     *
-     * \tparam KeyT             <b>[inferred]</b> Key type
-     * \tparam OffsetIteratorT  <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
-     */
-    template <
-        typename            KeyT,
-        typename            OffsetIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t SortKeys(
-        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        const KeyT          *d_keys_in,                             ///< [in] %Device-accessible pointer to the input data of key data to sort
-        KeyT                *d_keys_out,                            ///< [out] %Device-accessible pointer to the sorted output sequence of key data
-        int                 num_items,                              ///< [in] The total number of items to sort (across all segments)
-        int                 num_segments,                           ///< [in] The number of segments that comprise the sorting data
-        OffsetIteratorT     d_begin_offsets,                        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-        OffsetIteratorT     d_end_offsets,                          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
-        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
-        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        // Null value type
-        DoubleBuffer<KeyT>      d_keys(const_cast<KeyT*>(d_keys_in), d_keys_out);
-        DoubleBuffer<NullType>  d_values;
-
-        return DispatchSegmentedRadixSort<false, KeyT, NullType, OffsetIteratorT, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_keys,
-            d_values,
-            num_items,
-            num_segments,
-            d_begin_offsets,
-            d_end_offsets,
-            begin_bit,
-            end_bit,
-            false,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Sorts segments of keys into ascending order. (~<em>N </em>auxiliary storage required).
-     *
-     * \par
-     * - The sorting operation is given a pair of key buffers managed by a
-     *   DoubleBuffer structure that indicates which of the two buffers is
-     *   "current" (and thus contains the input data to be sorted).
-     * - The contents of both buffers may be altered by the sorting operation.
-     * - Upon completion, the sorting operation will update the "current" indicator
-     *   within the DoubleBuffer wrapper to reference which of the two buffers
-     *   now contains the sorted output sequence (a function of the number of key bits
-     *   specified and the targeted device architecture).
-     * - When input a contiguous sequence of segments, a single sequence
-     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
-     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
-     *   the latter is specified as <tt>segment_offsets+1</tt>).
-     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
-     * - \devicestorageP
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the batched sorting of three segments (with one zero-length segment) of \p int keys.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_segmentd_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for sorting data
-     * int  num_items;          // e.g., 7
-     * int  num_segments;       // e.g., 3
-     * int  *d_offsets;         // e.g., [0, 3, 3, 7]
-     * int  *d_key_buf;         // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_key_alt_buf;     // e.g., [-, -, -, -, -, -, -]
-     * ...
-     *
-     * // Create a DoubleBuffer to wrap the pair of device pointers
-     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSegmentedRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys,
-     *     num_items, num_segments, d_offsets, d_offsets + 1);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run sorting operation
-     * cub::DeviceSegmentedRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, d_keys,
-     *     num_items, num_segments, d_offsets, d_offsets + 1);
-     *
-     * // d_keys.Current()      <-- [6, 7, 8, 0, 3, 5, 9]
-     *
-     * \endcode
-     *
-     * \tparam KeyT             <b>[inferred]</b> Key type
-     * \tparam OffsetIteratorT  <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
-     */
-    template <
-        typename            KeyT,
-        typename            OffsetIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t SortKeys(
-        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        DoubleBuffer<KeyT>  &d_keys,                                ///< [in,out] Reference to the double-buffer of keys whose "current" device-accessible buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
-        int                 num_items,                              ///< [in] The total number of items to sort (across all segments)
-        int                 num_segments,                           ///< [in] The number of segments that comprise the sorting data
-        OffsetIteratorT     d_begin_offsets,                        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-        OffsetIteratorT     d_end_offsets,                          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
-        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
-        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        // Null value type
-        DoubleBuffer<NullType> d_values;
-
-        return DispatchSegmentedRadixSort<false, KeyT, NullType, OffsetIteratorT, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_keys,
-            d_values,
-            num_items,
-            num_segments,
-            d_begin_offsets,
-            d_end_offsets,
-            begin_bit,
-            end_bit,
-            true,
-            stream,
-            debug_synchronous);
-    }
-
-    /**
-     * \brief Sorts segments of keys into descending order. (~<em>2N</em> auxiliary storage required).
-     *
-     * \par
-     * - The contents of the input data are not altered by the sorting operation
-     * - When input a contiguous sequence of segments, a single sequence
-     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
-     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
-     *   the latter is specified as <tt>segment_offsets+1</tt>).
-     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
-     * - \devicestorageNP  For sorting using only <em>O</em>(<tt>P</tt>) temporary storage, see the sorting interface using DoubleBuffer wrappers below.
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the batched sorting of three segments (with one zero-length segment) of \p int keys.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_segmentd_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for sorting data
-     * int  num_items;          // e.g., 7
-     * int  num_segments;       // e.g., 3
-     * int  *d_offsets;         // e.g., [0, 3, 3, 7]
-     * int  *d_keys_in;         // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_keys_out;        // e.g., [-, -, -, -, -, -, -]
-     * ...
-     *
-     * // Create a DoubleBuffer to wrap the pair of device pointers
-     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSegmentedRadixSort::SortKeysDescending(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out,
-     *     num_items, num_segments, d_offsets, d_offsets + 1);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run sorting operation
-     * cub::DeviceSegmentedRadixSort::SortKeysDescending(d_temp_storage, temp_storage_bytes, d_keys_in, d_keys_out,
-     *     num_items, num_segments, d_offsets, d_offsets + 1);
-     *
-     * // d_keys_out            <-- [8, 7, 6, 9, 5, 3, 0]
-     *
-     * \endcode
-     *
-     * \tparam KeyT             <b>[inferred]</b> Key type
-     * \tparam OffsetIteratorT  <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
-     */
-    template <
-        typename            KeyT,
-        typename            OffsetIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t SortKeysDescending(
-        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        const KeyT          *d_keys_in,                             ///< [in] %Device-accessible pointer to the input data of key data to sort
-        KeyT                *d_keys_out,                            ///< [out] %Device-accessible pointer to the sorted output sequence of key data
-        int                 num_items,                              ///< [in] The total number of items to sort (across all segments)
-        int                 num_segments,                           ///< [in] The number of segments that comprise the sorting data
-        OffsetIteratorT     d_begin_offsets,                        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-        OffsetIteratorT     d_end_offsets,                          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
-        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
-        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        DoubleBuffer<KeyT>      d_keys(const_cast<KeyT*>(d_keys_in), d_keys_out);
-        DoubleBuffer<NullType>  d_values;
-
-        return DispatchSegmentedRadixSort<true, KeyT, NullType, OffsetIteratorT, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_keys,
-            d_values,
-            num_items,
-            num_segments,
-            d_begin_offsets,
-            d_end_offsets,
-            begin_bit,
-            end_bit,
-            false,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Sorts segments of keys into descending order. (~<em>N </em>auxiliary storage required).
-     *
-     * \par
-     * - The sorting operation is given a pair of key buffers managed by a
-     *   DoubleBuffer structure that indicates which of the two buffers is
-     *   "current" (and thus contains the input data to be sorted).
-     * - The contents of both buffers may be altered by the sorting operation.
-     * - Upon completion, the sorting operation will update the "current" indicator
-     *   within the DoubleBuffer wrapper to reference which of the two buffers
-     *   now contains the sorted output sequence (a function of the number of key bits
-     *   specified and the targeted device architecture).
-     * - When input a contiguous sequence of segments, a single sequence
-     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
-     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
-     *   the latter is specified as <tt>segment_offsets+1</tt>).
-     * - An optional bit subrange <tt>[begin_bit, end_bit)</tt> of differentiating key bits can be specified.  This can reduce overall sorting overhead and yield a corresponding performance improvement.
-     * - \devicestorageP
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the batched sorting of three segments (with one zero-length segment) of \p int keys.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_segmentd_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for sorting data
-     * int  num_items;          // e.g., 7
-     * int  num_segments;       // e.g., 3
-     * int  *d_offsets;         // e.g., [0, 3, 3, 7]
-     * int  *d_key_buf;         // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int  *d_key_alt_buf;     // e.g., [-, -, -, -, -, -, -]
-     * ...
-     *
-     * // Create a DoubleBuffer to wrap the pair of device pointers
-     * cub::DoubleBuffer<int> d_keys(d_key_buf, d_key_alt_buf);
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSegmentedRadixSort::SortKeysDescending(d_temp_storage, temp_storage_bytes, d_keys,
-     *     num_items, num_segments, d_offsets, d_offsets + 1);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run sorting operation
-     * cub::DeviceSegmentedRadixSort::SortKeysDescending(d_temp_storage, temp_storage_bytes, d_keys,
-     *     num_items, num_segments, d_offsets, d_offsets + 1);
-     *
-     * // d_keys.Current()      <-- [8, 7, 6, 9, 5, 3, 0]
-     *
-     * \endcode
-     *
-     * \tparam KeyT             <b>[inferred]</b> Key type
-     * \tparam OffsetIteratorT  <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
-     */
-    template <
-        typename            KeyT,
-        typename            OffsetIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t SortKeysDescending(
-        void                *d_temp_storage,                        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t              &temp_storage_bytes,                    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        DoubleBuffer<KeyT>  &d_keys,                                ///< [in,out] Reference to the double-buffer of keys whose "current" device-accessible buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
-        int                 num_items,                              ///< [in] The total number of items to sort (across all segments)
-        int                 num_segments,                           ///< [in] The number of segments that comprise the sorting data
-        OffsetIteratorT     d_begin_offsets,                        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-        OffsetIteratorT     d_end_offsets,                          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-        int                 begin_bit           = 0,                ///< [in] <b>[optional]</b> The least-significant bit index (inclusive)  needed for key comparison
-        int                 end_bit             = sizeof(KeyT) * 8, ///< [in] <b>[optional]</b> The most-significant bit index (exclusive) needed for key comparison (e.g., sizeof(unsigned int) * 8)
-        cudaStream_t        stream              = 0,                ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)            ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        // Null value type
-        DoubleBuffer<NullType> d_values;
-
-        return DispatchSegmentedRadixSort<true, KeyT, NullType, OffsetIteratorT, OffsetT>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_keys,
-            d_values,
-            num_items,
-            num_segments,
-            d_begin_offsets,
-            d_end_offsets,
-            begin_bit,
-            end_bit,
-            true,
-            stream,
-            debug_synchronous);
-    }
-
-
-    //@}  end member group
-
-
-};
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/device/device_segmented_reduce.cuh b/pfsimulator/third_party/cub/device/device_segmented_reduce.cuh
deleted file mode 100644
index 6c3b54a03..000000000
--- a/pfsimulator/third_party/cub/device/device_segmented_reduce.cuh
+++ /dev/null
@@ -1,619 +0,0 @@
-
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::DeviceSegmentedReduce provides device-wide, parallel operations for computing a batched reduction across multiple sequences of data items residing within device-accessible memory.
- */
-
-#pragma once
-
-#include <stdio.h>
-#include <iterator>
-
-#include "../iterator/arg_index_input_iterator.cuh"
-#include "dispatch/dispatch_reduce.cuh"
-#include "dispatch/dispatch_reduce_by_key.cuh"
-#include "../util_type.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \brief DeviceSegmentedReduce provides device-wide, parallel operations for computing a reduction across multiple sequences of data items residing within device-accessible memory. ![](reduce_logo.png)
- * \ingroup SegmentedModule
- *
- * \par Overview
- * A <a href="http://en.wikipedia.org/wiki/Reduce_(higher-order_function)"><em>reduction</em></a> (or <em>fold</em>)
- * uses a binary combining operator to compute a single aggregate from a sequence of input elements.
- *
- * \par Usage Considerations
- * \cdp_class{DeviceSegmentedReduce}
- *
- */
-struct DeviceSegmentedReduce
-{
-    /**
-     * \brief Computes a device-wide segmented reduction using the specified binary \p reduction_op functor.
-     *
-     * \par
-     * - Does not support binary reduction operators that are non-commutative.
-     * - When input a contiguous sequence of segments, a single sequence
-     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
-     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
-     *   the latter is specified as <tt>segment_offsets+1</tt>).
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates a custom min-reduction of a device vector of \p int data elements.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
-     *
-     * // CustomMin functor
-     * struct CustomMin
-     * {
-     *     template <typename T>
-     *     CUB_RUNTIME_FUNCTION __forceinline__
-     *     T operator()(const T &a, const T &b) const {
-     *         return (b < a) ? b : a;
-     *     }
-     * };
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int          num_segments;   // e.g., 3
-     * int          *d_offsets;     // e.g., [0, 3, 3, 7]
-     * int          *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int          *d_out;         // e.g., [-, -, -]
-     * CustomMin    min_op;
-     * int          initial_value;           // e.g., INT_MAX
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSegmentedReduce::Reduce(d_temp_storage, temp_storage_bytes, d_in, d_out,
-     *     num_segments, d_offsets, d_offsets + 1, min_op, initial_value);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run reduction
-     * cub::DeviceSegmentedReduce::Reduce(d_temp_storage, temp_storage_bytes, d_in, d_out,
-     *     num_segments, d_offsets, d_offsets + 1, min_op, initial_value);
-     *
-     * // d_out <-- [6, INT_MAX, 0]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT       <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
-     * \tparam OutputIteratorT      <b>[inferred]</b> Output iterator type for recording the reduced aggregate \iterator
-     * \tparam OffsetIteratorT      <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
-     * \tparam ReductionOp          <b>[inferred]</b> Binary reduction functor type having member <tt>T operator()(const T &a, const T &b)</tt>
-     * \tparam T                    <b>[inferred]</b> Data element type that is convertible to the \p value type of \p InputIteratorT
-     */
-    template <
-        typename            InputIteratorT,
-        typename            OutputIteratorT,
-        typename            OffsetIteratorT,
-        typename            ReductionOp,
-        typename            T>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t Reduce(
-        void                *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t              &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT      d_in,                               ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT     d_out,                              ///< [out] Pointer to the output aggregate
-        int                 num_segments,                       ///< [in] The number of segments that comprise the sorting data
-        OffsetIteratorT     d_begin_offsets,                    ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-        OffsetIteratorT     d_end_offsets,                      ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-        ReductionOp         reduction_op,                       ///< [in] Binary reduction functor 
-        T                   initial_value,                      ///< [in] Initial value of the reduction for each segment
-        cudaStream_t        stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        return DispatchSegmentedReduce<InputIteratorT, OutputIteratorT, OffsetIteratorT, OffsetT, ReductionOp>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            d_out,
-            num_segments,
-            d_begin_offsets,
-            d_end_offsets,
-            reduction_op,
-            initial_value,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Computes a device-wide segmented sum using the addition ('+') operator.
-     *
-     * \par
-     * - Uses \p 0 as the initial value of the reduction for each segment.
-     * - When input a contiguous sequence of segments, a single sequence
-     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
-     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
-     *   the latter is specified as <tt>segment_offsets+1</tt>).
-     * - Does not support \p + operators that are non-commutative..
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the sum reduction of a device vector of \p int data elements.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int num_segments;   // e.g., 3
-     * int *d_offsets;     // e.g., [0, 3, 3, 7]
-     * int *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int *d_out;         // e.g., [-, -, -]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSegmentedReduce::Sum(d_temp_storage, temp_storage_bytes, d_in, d_out,
-     *     num_segments, d_offsets, d_offsets + 1);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run sum-reduction
-     * cub::DeviceSegmentedReduce::Sum(d_temp_storage, temp_storage_bytes, d_in, d_out,
-     *     num_segments, d_offsets, d_offsets + 1);
-     *
-     * // d_out <-- [21, 0, 17]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
-     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate \iterator
-     * \tparam OffsetIteratorT      <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
-     */
-    template <
-        typename            InputIteratorT,
-        typename            OutputIteratorT,
-        typename            OffsetIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t Sum(
-        void                *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t              &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT      d_in,                               ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT     d_out,                              ///< [out] Pointer to the output aggregate
-        int                 num_segments,                       ///< [in] The number of segments that comprise the sorting data
-        OffsetIteratorT     d_begin_offsets,                    ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-        OffsetIteratorT     d_end_offsets,                      ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-        cudaStream_t        stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        // The output value type
-        typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
-            typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
-            typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
-
-        return DispatchSegmentedReduce<InputIteratorT,  OutputIteratorT, OffsetIteratorT, OffsetT, cub::Sum>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            d_out,
-            num_segments,
-            d_begin_offsets,
-            d_end_offsets,
-            cub::Sum(),
-            OutputT(),            // zero-initialize
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Computes a device-wide segmented minimum using the less-than ('<') operator.
-     *
-     * \par
-     * - Uses <tt>std::numeric_limits<T>::max()</tt> as the initial value of the reduction for each segment.
-     * - When input a contiguous sequence of segments, a single sequence
-     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
-     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
-     *   the latter is specified as <tt>segment_offsets+1</tt>).
-     * - Does not support \p < operators that are non-commutative.
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the min-reduction of a device vector of \p int data elements.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int num_segments;   // e.g., 3
-     * int *d_offsets;     // e.g., [0, 3, 3, 7]
-     * int *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int *d_out;         // e.g., [-, -, -]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSegmentedReduce::Min(d_temp_storage, temp_storage_bytes, d_in, d_out,
-     *     num_segments, d_offsets, d_offsets + 1);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run min-reduction
-     * cub::DeviceSegmentedReduce::Min(d_temp_storage, temp_storage_bytes, d_in, d_out,
-     *     num_segments, d_offsets, d_offsets + 1);
-     *
-     * // d_out <-- [6, INT_MAX, 0]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
-     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate \iterator
-     * \tparam OffsetIteratorT      <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
-     */
-    template <
-        typename            InputIteratorT,
-        typename            OutputIteratorT,
-        typename            OffsetIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t Min(
-        void                *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t              &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT      d_in,                               ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT     d_out,                              ///< [out] Pointer to the output aggregate
-        int                 num_segments,                       ///< [in] The number of segments that comprise the sorting data
-        OffsetIteratorT     d_begin_offsets,                    ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-        OffsetIteratorT     d_end_offsets,                      ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-        cudaStream_t        stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        // The input value type
-        typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
-
-        return DispatchSegmentedReduce<InputIteratorT,  OutputIteratorT, OffsetIteratorT, OffsetT, cub::Min>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            d_out,
-            num_segments,
-            d_begin_offsets,
-            d_end_offsets,
-            cub::Min(),
-            Traits<InputT>::Max(),    // replace with std::numeric_limits<T>::max() when C++11 support is more prevalent
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Finds the first device-wide minimum in each segment using the less-than ('<') operator, also returning the in-segment index of that item.
-     *
-     * \par
-     * - The output value type of \p d_out is cub::KeyValuePair <tt><int, T></tt> (assuming the value type of \p d_in is \p T)
-     *   - The minimum of the <em>i</em><sup>th</sup> segment is written to <tt>d_out[i].value</tt> and its offset in that segment is written to <tt>d_out[i].key</tt>.
-     *   - The <tt>{1, std::numeric_limits<T>::max()}</tt> tuple is produced for zero-length inputs
-     * - When input a contiguous sequence of segments, a single sequence
-     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
-     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
-     *   the latter is specified as <tt>segment_offsets+1</tt>).
-     * - Does not support \p < operators that are non-commutative.
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the argmin-reduction of a device vector of \p int data elements.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int                      num_segments;   // e.g., 3
-     * int                      *d_offsets;     // e.g., [0, 3, 3, 7]
-     * int                      *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * KeyValuePair<int, int>   *d_out;         // e.g., [{-,-}, {-,-}, {-,-}]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSegmentedReduce::ArgMin(d_temp_storage, temp_storage_bytes, d_in, d_out,
-     *     num_segments, d_offsets, d_offsets + 1);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run argmin-reduction
-     * cub::DeviceSegmentedReduce::ArgMin(d_temp_storage, temp_storage_bytes, d_in, d_out,
-     *     num_segments, d_offsets, d_offsets + 1);
-     *
-     * // d_out <-- [{1,6}, {1,INT_MAX}, {2,0}]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items (of some type \p T) \iterator
-     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate (having value type <tt>KeyValuePair<int, T></tt>) \iterator
-     * \tparam OffsetIteratorT      <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
-     */
-    template <
-        typename            InputIteratorT,
-        typename            OutputIteratorT,
-        typename            OffsetIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t ArgMin(
-        void                *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t              &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT      d_in,                               ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT     d_out,                              ///< [out] Pointer to the output aggregate
-        int                 num_segments,                       ///< [in] The number of segments that comprise the sorting data
-        OffsetIteratorT     d_begin_offsets,                    ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-        OffsetIteratorT     d_end_offsets,                      ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-        cudaStream_t        stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        // The input type
-        typedef typename std::iterator_traits<InputIteratorT>::value_type InputValueT;
-
-        // The output tuple type
-        typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
-            KeyValuePair<OffsetT, InputValueT>,                                                                 // ... then the key value pair OffsetT + InputValueT
-            typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputTupleT;                     // ... else the output iterator's value type
-
-        // The output value type
-        typedef typename OutputTupleT::Value OutputValueT;
-
-        // Wrapped input iterator to produce index-value <OffsetT, InputT> tuples
-        typedef ArgIndexInputIterator<InputIteratorT, OffsetT, OutputValueT> ArgIndexInputIteratorT;
-        ArgIndexInputIteratorT d_indexed_in(d_in);
-
-        // Initial value
-        OutputTupleT initial_value(1, Traits<InputValueT>::Max());   // replace with std::numeric_limits<T>::max() when C++11 support is more prevalent
-
-        return DispatchSegmentedReduce<ArgIndexInputIteratorT,  OutputIteratorT, OffsetIteratorT, OffsetT, cub::ArgMin>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_indexed_in,
-            d_out,
-            num_segments,
-            d_begin_offsets,
-            d_end_offsets,
-            cub::ArgMin(),
-            initial_value,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Computes a device-wide segmented maximum using the greater-than ('>') operator.
-     *
-     * \par
-     * - Uses <tt>std::numeric_limits<T>::lowest()</tt> as the initial value of the reduction.
-     * - When input a contiguous sequence of segments, a single sequence
-     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
-     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
-     *   the latter is specified as <tt>segment_offsets+1</tt>).
-     * - Does not support \p > operators that are non-commutative.
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the max-reduction of a device vector of \p int data elements.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_radix_sort.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int num_segments;   // e.g., 3
-     * int *d_offsets;     // e.g., [0, 3, 3, 7]
-     * int *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * int *d_out;         // e.g., [-, -, -]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSegmentedReduce::Max(d_temp_storage, temp_storage_bytes, d_in, d_out,
-     *     num_segments, d_offsets, d_offsets + 1);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run max-reduction
-     * cub::DeviceSegmentedReduce::Max(d_temp_storage, temp_storage_bytes, d_in, d_out,
-     *     num_segments, d_offsets, d_offsets + 1);
-     *
-     * // d_out <-- [8, INT_MIN, 9]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
-     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate \iterator
-     * \tparam OffsetIteratorT      <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
-     */
-    template <
-        typename            InputIteratorT,
-        typename            OutputIteratorT,
-        typename            OffsetIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t Max(
-        void                *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t              &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT      d_in,                               ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT     d_out,                              ///< [out] Pointer to the output aggregate
-        int                 num_segments,                       ///< [in] The number of segments that comprise the sorting data
-        OffsetIteratorT     d_begin_offsets,                    ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-        OffsetIteratorT     d_end_offsets,                      ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-        cudaStream_t        stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        // The input value type
-        typedef typename std::iterator_traits<InputIteratorT>::value_type InputT;
-
-        return DispatchSegmentedReduce<InputIteratorT,  OutputIteratorT, OffsetIteratorT, OffsetT, cub::Max>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            d_out,
-            num_segments,
-            d_begin_offsets,
-            d_end_offsets,
-            cub::Max(),
-            Traits<InputT>::Lowest(),    // replace with std::numeric_limits<T>::lowest() when C++11 support is more prevalent
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Finds the first device-wide maximum in each segment using the greater-than ('>') operator, also returning the in-segment index of that item
-     *
-     * \par
-     * - The output value type of \p d_out is cub::KeyValuePair <tt><int, T></tt> (assuming the value type of \p d_in is \p T)
-     *   - The maximum of the <em>i</em><sup>th</sup> segment is written to <tt>d_out[i].value</tt> and its offset in that segment is written to <tt>d_out[i].key</tt>.
-     *   - The <tt>{1, std::numeric_limits<T>::lowest()}</tt> tuple is produced for zero-length inputs
-     * - When input a contiguous sequence of segments, a single sequence
-     *   \p segment_offsets (of length <tt>num_segments+1</tt>) can be aliased
-     *   for both the \p d_begin_offsets and \p d_end_offsets parameters (where
-     *   the latter is specified as <tt>segment_offsets+1</tt>).
-     * - Does not support \p > operators that are non-commutative.
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the argmax-reduction of a device vector of \p int data elements.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_reduce.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int                      num_segments;   // e.g., 3
-     * int                      *d_offsets;     // e.g., [0, 3, 3, 7]
-     * int                      *d_in;          // e.g., [8, 6, 7, 5, 3, 0, 9]
-     * KeyValuePair<int, int>   *d_out;         // e.g., [{-,-}, {-,-}, {-,-}]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSegmentedReduce::ArgMax(d_temp_storage, temp_storage_bytes, d_in, d_out,
-     *     num_segments, d_offsets, d_offsets + 1);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run argmax-reduction
-     * cub::DeviceSegmentedReduce::ArgMax(d_temp_storage, temp_storage_bytes, d_in, d_out,
-     *     num_segments, d_offsets, d_offsets + 1);
-     *
-     * // d_out <-- [{0,8}, {1,INT_MIN}, {3,9}]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT     <b>[inferred]</b> Random-access input iterator type for reading input items (of some type \p T) \iterator
-     * \tparam OutputIteratorT    <b>[inferred]</b> Output iterator type for recording the reduced aggregate (having value type <tt>KeyValuePair<int, T></tt>) \iterator
-     * \tparam OffsetIteratorT    <b>[inferred]</b> Random-access input iterator type for reading segment offsets \iterator
-     */
-    template <
-        typename            InputIteratorT,
-        typename            OutputIteratorT,
-        typename            OffsetIteratorT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t ArgMax(
-        void                *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t              &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT      d_in,                               ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT     d_out,                              ///< [out] Pointer to the output aggregate
-        int                 num_segments,                       ///< [in] The number of segments that comprise the sorting data
-        OffsetIteratorT     d_begin_offsets,                    ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-        OffsetIteratorT     d_end_offsets,                      ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-        cudaStream_t        stream              = 0,            ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous   = false)        ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        // Signed integer type for global offsets
-        typedef int OffsetT;
-
-        // The input type
-        typedef typename std::iterator_traits<InputIteratorT>::value_type InputValueT;
-
-        // The output tuple type
-        typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
-            KeyValuePair<OffsetT, InputValueT>,                                                                 // ... then the key value pair OffsetT + InputValueT
-            typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputTupleT;                     // ... else the output iterator's value type
-
-        // The output value type
-        typedef typename OutputTupleT::Value OutputValueT;
-
-        // Wrapped input iterator to produce index-value <OffsetT, InputT> tuples
-        typedef ArgIndexInputIterator<InputIteratorT, OffsetT, OutputValueT> ArgIndexInputIteratorT;
-        ArgIndexInputIteratorT d_indexed_in(d_in);
-
-        // Initial value
-        OutputTupleT initial_value(1, Traits<InputValueT>::Lowest());     // replace with std::numeric_limits<T>::lowest() when C++11 support is more prevalent
-
-        return DispatchSegmentedReduce<ArgIndexInputIteratorT, OutputIteratorT, OffsetIteratorT, OffsetT, cub::ArgMax>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_indexed_in,
-            d_out,
-            num_segments,
-            d_begin_offsets,
-            d_end_offsets,
-            cub::ArgMax(),
-            initial_value,
-            stream,
-            debug_synchronous);
-    }
-
-};
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/device/device_select.cuh b/pfsimulator/third_party/cub/device/device_select.cuh
deleted file mode 100644
index 52a3e126d..000000000
--- a/pfsimulator/third_party/cub/device/device_select.cuh
+++ /dev/null
@@ -1,369 +0,0 @@
-
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::DeviceSelect provides device-wide, parallel operations for compacting selected items from sequences of data items residing within device-accessible memory.
- */
-
-#pragma once
-
-#include <stdio.h>
-#include <iterator>
-
-#include "dispatch/dispatch_select_if.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \brief DeviceSelect provides device-wide, parallel operations for compacting selected items from sequences of data items residing within device-accessible memory. ![](select_logo.png)
- * \ingroup SingleModule
- *
- * \par Overview
- * These operations apply a selection criterion to selectively copy
- * items from a specified input sequence to a compact output sequence.
- *
- * \par Usage Considerations
- * \cdp_class{DeviceSelect}
- *
- * \par Performance
- * \linear_performance{select-flagged, select-if, and select-unique}
- *
- * \par
- * The following chart illustrates DeviceSelect::If
- * performance across different CUDA architectures for \p int32 items,
- * where 50% of the items are randomly selected.
- *
- * \image html select_if_int32_50_percent.png
- *
- * \par
- * The following chart illustrates DeviceSelect::Unique
- * performance across different CUDA architectures for \p int32 items
- * where segments have lengths uniformly sampled from [1,1000].
- *
- * \image html select_unique_int32_len_500.png
- *
- * \par
- * \plots_below
- *
- */
-struct DeviceSelect
-{
-    /**
-     * \brief Uses the \p d_flags sequence to selectively copy the corresponding items from \p d_in into \p d_out.  The total number of items selected is written to \p d_num_selected_out. ![](select_flags_logo.png)
-     *
-     * \par
-     * - The value type of \p d_flags must be castable to \p bool (e.g., \p bool, \p char, \p int, etc.).
-     * - Copies of the selected items are compacted into \p d_out and maintain their original relative ordering.
-     * - \devicestorage
-     *
-     * \par Snippet
-     * The code snippet below illustrates the compaction of items selected from an \p int device vector.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>       // or equivalently <cub/device/device_select.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input, flags, and output
-     * int  num_items;              // e.g., 8
-     * int  *d_in;                  // e.g., [1, 2, 3, 4, 5, 6, 7, 8]
-     * char *d_flags;               // e.g., [1, 0, 0, 1, 0, 1, 1, 0]
-     * int  *d_out;                 // e.g., [ ,  ,  ,  ,  ,  ,  ,  ]
-     * int  *d_num_selected_out;    // e.g., [ ]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSelect::Flagged(d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run selection
-     * cub::DeviceSelect::Flagged(d_temp_storage, temp_storage_bytes, d_in, d_flags, d_out, d_num_selected_out, num_items);
-     *
-     * // d_out                 <-- [1, 4, 6, 7]
-     * // d_num_selected_out    <-- [4]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT       <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
-     * \tparam FlagIterator         <b>[inferred]</b> Random-access input iterator type for reading selection flags \iterator
-     * \tparam OutputIteratorT      <b>[inferred]</b> Random-access output iterator type for writing selected items \iterator
-     * \tparam NumSelectedIteratorT  <b>[inferred]</b> Output iterator type for recording the number of items selected \iterator
-     */
-    template <
-        typename                    InputIteratorT,
-        typename                    FlagIterator,
-        typename                    OutputIteratorT,
-        typename                    NumSelectedIteratorT>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Flagged(
-        void*               d_temp_storage,                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                      &temp_storage_bytes,            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT              d_in,                           ///< [in] Pointer to the input sequence of data items
-        FlagIterator                d_flags,                        ///< [in] Pointer to the input sequence of selection flags
-        OutputIteratorT             d_out,                          ///< [out] Pointer to the output sequence of selected data items
-        NumSelectedIteratorT         d_num_selected_out,                 ///< [out] Pointer to the output total number of items selected (i.e., length of \p d_out)
-        int                         num_items,                      ///< [in] Total number of input items (i.e., length of \p d_in)
-        cudaStream_t                stream             = 0,         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous  = false)     ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        typedef int                     OffsetT;         // Signed integer type for global offsets
-        typedef NullType                SelectOp;       // Selection op (not used)
-        typedef NullType                EqualityOp;     // Equality operator (not used)
-
-        return DispatchSelectIf<InputIteratorT, FlagIterator, OutputIteratorT, NumSelectedIteratorT, SelectOp, EqualityOp, OffsetT, false>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            d_flags,
-            d_out,
-            d_num_selected_out,
-            SelectOp(),
-            EqualityOp(),
-            num_items,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Uses the \p select_op functor to selectively copy items from \p d_in into \p d_out.  The total number of items selected is written to \p d_num_selected_out. ![](select_logo.png)
-     *
-     * \par
-     * - Copies of the selected items are compacted into \p d_out and maintain their original relative ordering.
-     * - \devicestorage
-     *
-     * \par Performance
-     * The following charts illustrate saturated select-if performance across different
-     * CUDA architectures for \p int32 and \p int64 items, respectively.  Items are
-     * selected with 50% probability.
-     *
-     * \image html select_if_int32_50_percent.png
-     * \image html select_if_int64_50_percent.png
-     *
-     * \par
-     * The following charts are similar, but 5% selection probability:
-     *
-     * \image html select_if_int32_5_percent.png
-     * \image html select_if_int64_5_percent.png
-     *
-     * \par Snippet
-     * The code snippet below illustrates the compaction of items selected from an \p int device vector.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_select.cuh>
-     *
-     * // Functor type for selecting values less than some criteria
-     * struct LessThan
-     * {
-     *     int compare;
-     *
-     *     CUB_RUNTIME_FUNCTION __forceinline__
-     *     LessThan(int compare) : compare(compare) {}
-     *
-     *     CUB_RUNTIME_FUNCTION __forceinline__
-     *     bool operator()(const int &a) const {
-     *         return (a < compare);
-     *     }
-     * };
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int      num_items;              // e.g., 8
-     * int      *d_in;                  // e.g., [0, 2, 3, 9, 5, 2, 81, 8]
-     * int      *d_out;                 // e.g., [ ,  ,  ,  ,  ,  ,  ,  ]
-     * int      *d_num_selected_out;    // e.g., [ ]
-     * LessThan select_op(7);
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSelect::If(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, select_op);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run selection
-     * cub::DeviceSelect::If(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items, select_op);
-     *
-     * // d_out                 <-- [0, 2, 3, 5, 2]
-     * // d_num_selected_out    <-- [5]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT       <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
-     * \tparam OutputIteratorT      <b>[inferred]</b> Random-access output iterator type for writing selected items \iterator
-     * \tparam NumSelectedIteratorT  <b>[inferred]</b> Output iterator type for recording the number of items selected \iterator
-     * \tparam SelectOp             <b>[inferred]</b> Selection operator type having member <tt>bool operator()(const T &a)</tt>
-     */
-    template <
-        typename                    InputIteratorT,
-        typename                    OutputIteratorT,
-        typename                    NumSelectedIteratorT,
-        typename                    SelectOp>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t If(
-        void*               d_temp_storage,                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                      &temp_storage_bytes,            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT              d_in,                           ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT             d_out,                          ///< [out] Pointer to the output sequence of selected data items
-        NumSelectedIteratorT         d_num_selected_out,                 ///< [out] Pointer to the output total number of items selected (i.e., length of \p d_out)
-        int                         num_items,                      ///< [in] Total number of input items (i.e., length of \p d_in)
-        SelectOp                    select_op,                      ///< [in] Unary selection operator
-        cudaStream_t                stream             = 0,         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous  = false)     ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        typedef int                     OffsetT;         // Signed integer type for global offsets
-        typedef NullType*               FlagIterator;   // FlagT iterator type (not used)
-        typedef NullType                EqualityOp;     // Equality operator (not used)
-
-        return DispatchSelectIf<InputIteratorT, FlagIterator, OutputIteratorT, NumSelectedIteratorT, SelectOp, EqualityOp, OffsetT, false>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            NULL,
-            d_out,
-            d_num_selected_out,
-            select_op,
-            EqualityOp(),
-            num_items,
-            stream,
-            debug_synchronous);
-    }
-
-
-    /**
-     * \brief Given an input sequence \p d_in having runs of consecutive equal-valued keys, only the first key from each run is selectively copied to \p d_out.  The total number of items selected is written to \p d_num_selected_out. ![](unique_logo.png)
-     *
-     * \par
-     * - The <tt>==</tt> equality operator is used to determine whether keys are equivalent
-     * - Copies of the selected items are compacted into \p d_out and maintain their original relative ordering.
-     * - \devicestorage
-     *
-     * \par Performance
-     * The following charts illustrate saturated select-unique performance across different
-     * CUDA architectures for \p int32 and \p int64 items, respectively.  Segments have
-     * lengths uniformly sampled from [1,1000].
-     *
-     * \image html select_unique_int32_len_500.png
-     * \image html select_unique_int64_len_500.png
-     *
-     * \par
-     * The following charts are similar, but with segment lengths uniformly sampled from [1,10]:
-     *
-     * \image html select_unique_int32_len_5.png
-     * \image html select_unique_int64_len_5.png
-     *
-     * \par Snippet
-     * The code snippet below illustrates the compaction of items selected from an \p int device vector.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>       // or equivalently <cub/device/device_select.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input and output
-     * int  num_items;              // e.g., 8
-     * int  *d_in;                  // e.g., [0, 2, 2, 9, 5, 5, 5, 8]
-     * int  *d_out;                 // e.g., [ ,  ,  ,  ,  ,  ,  ,  ]
-     * int  *d_num_selected_out;    // e.g., [ ]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void     *d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSelect::Unique(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run selection
-     * cub::DeviceSelect::Unique(d_temp_storage, temp_storage_bytes, d_in, d_out, d_num_selected_out, num_items);
-     *
-     * // d_out                 <-- [0, 2, 9, 5, 8]
-     * // d_num_selected_out    <-- [5]
-     *
-     * \endcode
-     *
-     * \tparam InputIteratorT       <b>[inferred]</b> Random-access input iterator type for reading input items \iterator
-     * \tparam OutputIteratorT      <b>[inferred]</b> Random-access output iterator type for writing selected items \iterator
-     * \tparam NumSelectedIteratorT  <b>[inferred]</b> Output iterator type for recording the number of items selected \iterator
-     */
-    template <
-        typename                    InputIteratorT,
-        typename                    OutputIteratorT,
-        typename                    NumSelectedIteratorT>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Unique(
-        void*               d_temp_storage,                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                      &temp_storage_bytes,            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT              d_in,                           ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT             d_out,                          ///< [out] Pointer to the output sequence of selected data items
-        NumSelectedIteratorT         d_num_selected_out,             ///< [out] Pointer to the output total number of items selected (i.e., length of \p d_out)
-        int                         num_items,                      ///< [in] Total number of input items (i.e., length of \p d_in)
-        cudaStream_t                stream             = 0,         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous  = false)     ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        typedef int                     OffsetT;         // Signed integer type for global offsets
-        typedef NullType*               FlagIterator;   // FlagT iterator type (not used)
-        typedef NullType                SelectOp;       // Selection op (not used)
-        typedef Equality                EqualityOp;     // Default == operator
-
-        return DispatchSelectIf<InputIteratorT, FlagIterator, OutputIteratorT, NumSelectedIteratorT, SelectOp, EqualityOp, OffsetT, false>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            d_in,
-            NULL,
-            d_out,
-            d_num_selected_out,
-            SelectOp(),
-            EqualityOp(),
-            num_items,
-            stream,
-            debug_synchronous);
-    }
-
-};
-
-/**
- * \example example_device_select_flagged.cu
- * \example example_device_select_if.cu
- * \example example_device_select_unique.cu
- */
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/device/device_spmv.cuh b/pfsimulator/third_party/cub/device/device_spmv.cuh
deleted file mode 100644
index 63b6a7e86..000000000
--- a/pfsimulator/third_party/cub/device/device_spmv.cuh
+++ /dev/null
@@ -1,174 +0,0 @@
-
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::DeviceSpmv provides device-wide parallel operations for performing sparse-matrix * vector multiplication (SpMV).
- */
-
-#pragma once
-
-#include <stdio.h>
-#include <iterator>
-#include <limits>
-
-#include "dispatch/dispatch_spmv_orig.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \brief DeviceSpmv provides device-wide parallel operations for performing sparse-matrix * dense-vector multiplication (SpMV).
- * \ingroup SingleModule
- *
- * \par Overview
- * The [<em>SpMV computation</em>](http://en.wikipedia.org/wiki/Sparse_matrix-vector_multiplication)
- * performs the matrix-vector operation
- * <em>y</em> = <em>alpha</em>*<b>A</b>*<em>x</em> + <em>beta</em>*<em>y</em>,
- * where:
- *  - <b>A</b> is an <em>m</em>x<em>n</em> sparse matrix whose non-zero structure is specified in
- *    [<em>compressed-storage-row (CSR) format</em>](http://en.wikipedia.org/wiki/Sparse_matrix#Compressed_row_Storage_.28CRS_or_CSR.29)
- *    (i.e., three arrays: <em>values</em>, <em>row_offsets</em>, and <em>column_indices</em>)
- *  - <em>x</em> and <em>y</em> are dense vectors
- *  - <em>alpha</em> and <em>beta</em> are scalar multiplicands
- *
- * \par Usage Considerations
- * \cdp_class{DeviceSpmv}
- *
- */
-struct DeviceSpmv
-{
-    /******************************************************************//**
-     * \name CSR matrix operations
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief This function performs the matrix-vector operation <em>y</em> = <b>A</b>*<em>x</em>.
-     *
-     * \par Snippet
-     * The code snippet below illustrates SpMV upon a 9x9 CSR matrix <b>A</b>
-     * representing a 3x3 lattice (24 non-zeros).
-     *
-     * \par
-     * \code
-     * #include <cub/cub.cuh>   // or equivalently <cub/device/device_spmv.cuh>
-     *
-     * // Declare, allocate, and initialize device-accessible pointers for input matrix A, input vector x,
-     * // and output vector y
-     * int    num_rows = 9;
-     * int    num_cols = 9;
-     * int    num_nonzeros = 24;
-     *
-     * float* d_values;  // e.g., [1, 1, 1, 1, 1, 1, 1, 1,
-     *                   //        1, 1, 1, 1, 1, 1, 1, 1,
-     *                   //        1, 1, 1, 1, 1, 1, 1, 1]
-     *
-     * int*   d_column_indices; // e.g., [1, 3, 0, 2, 4, 1, 5, 0,
-     *                          //        4, 6, 1, 3, 5, 7, 2, 4,
-     *                          //        8, 3, 7, 4, 6, 8, 5, 7]
-     *
-     * int*   d_row_offsets;    // e.g., [0, 2, 5, 7, 10, 14, 17, 19, 22, 24]
-     *
-     * float* d_vector_x;       // e.g., [1, 1, 1, 1, 1, 1, 1, 1, 1]
-     * float* d_vector_y;       // e.g., [ ,  ,  ,  ,  ,  ,  ,  ,  ]
-     * ...
-     *
-     * // Determine temporary device storage requirements
-     * void*    d_temp_storage = NULL;
-     * size_t   temp_storage_bytes = 0;
-     * cub::DeviceSpmv::CsrMV(d_temp_storage, temp_storage_bytes, d_values,
-     *     d_row_offsets, d_column_indices, d_vector_x, d_vector_y,
-     *     num_rows, num_cols, num_nonzeros, alpha, beta);
-     *
-     * // Allocate temporary storage
-     * cudaMalloc(&d_temp_storage, temp_storage_bytes);
-     *
-     * // Run SpMV
-     * cub::DeviceSpmv::CsrMV(d_temp_storage, temp_storage_bytes, d_values,
-     *     d_row_offsets, d_column_indices, d_vector_x, d_vector_y,
-     *     num_rows, num_cols, num_nonzeros, alpha, beta);
-     *
-     * // d_vector_y <-- [2, 3, 2, 3, 4, 3, 2, 3, 2]
-     *
-     * \endcode
-     *
-     * \tparam ValueT       <b>[inferred]</b> Matrix and vector value type (e.g., /p float, /p double, etc.)
-     */
-    template <
-        typename            ValueT>
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t CsrMV(
-        void*               d_temp_storage,                     ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&             temp_storage_bytes,                 ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        ValueT*             d_values,                           ///< [in] Pointer to the array of \p num_nonzeros values of the corresponding nonzero elements of matrix <b>A</b>.
-        int*                d_row_offsets,                      ///< [in] Pointer to the array of \p m + 1 offsets demarcating the start of every row in \p d_column_indices and \p d_values (with the final entry being equal to \p num_nonzeros)
-        int*                d_column_indices,                   ///< [in] Pointer to the array of \p num_nonzeros column-indices of the corresponding nonzero elements of matrix <b>A</b>.  (Indices are zero-valued.)
-        ValueT*             d_vector_x,                         ///< [in] Pointer to the array of \p num_cols values corresponding to the dense input vector <em>x</em>
-        ValueT*             d_vector_y,                         ///< [out] Pointer to the array of \p num_rows values corresponding to the dense output vector <em>y</em>
-        int                 num_rows,                           ///< [in] number of rows of matrix <b>A</b>.
-        int                 num_cols,                           ///< [in] number of columns of matrix <b>A</b>.
-        int                 num_nonzeros,                       ///< [in] number of nonzero elements of matrix <b>A</b>.
-        cudaStream_t        stream                  = 0,        ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous       = false)    ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        SpmvParams<ValueT, int> spmv_params;
-        spmv_params.d_values             = d_values;
-        spmv_params.d_row_end_offsets    = d_row_offsets + 1;
-        spmv_params.d_column_indices     = d_column_indices;
-        spmv_params.d_vector_x           = d_vector_x;
-        spmv_params.d_vector_y           = d_vector_y;
-        spmv_params.num_rows             = num_rows;
-        spmv_params.num_cols             = num_cols;
-        spmv_params.num_nonzeros         = num_nonzeros;
-        spmv_params.alpha                = 1.0;
-        spmv_params.beta                 = 0.0;
-
-        return DispatchSpmv<ValueT, int>::Dispatch(
-            d_temp_storage,
-            temp_storage_bytes,
-            spmv_params,
-            stream,
-            debug_synchronous);
-    }
-
-    //@}  end member group
-};
-
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/device/dispatch/dispatch_histogram.cuh b/pfsimulator/third_party/cub/device/dispatch/dispatch_histogram.cuh
deleted file mode 100644
index ab08e8ed0..000000000
--- a/pfsimulator/third_party/cub/device/dispatch/dispatch_histogram.cuh
+++ /dev/null
@@ -1,1096 +0,0 @@
-
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::DeviceHistogram provides device-wide parallel operations for constructing histogram(s) from a sequence of samples data residing within device-accessible memory.
- */
-
-#pragma once
-
-#include <stdio.h>
-#include <iterator>
-#include <limits>
-
-#include "../../agent/agent_histogram.cuh"
-#include "../../util_debug.cuh"
-#include "../../util_device.cuh"
-#include "../../thread/thread_search.cuh"
-#include "../../grid/grid_queue.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-
-/******************************************************************************
- * Histogram kernel entry points
- *****************************************************************************/
-
-/**
- * Histogram initialization kernel entry point
- */
-template <
-    int                                             NUM_ACTIVE_CHANNELS,            ///< Number of channels actively being histogrammed
-    typename                                        CounterT,                       ///< Integer type for counting sample occurrences per histogram bin
-    typename                                        OffsetT>                        ///< Signed integer type for global offsets
-__global__ void DeviceHistogramInitKernel(
-    ArrayWrapper<int, NUM_ACTIVE_CHANNELS>          num_output_bins_wrapper,        ///< Number of output histogram bins per channel
-    ArrayWrapper<CounterT*, NUM_ACTIVE_CHANNELS>    d_output_histograms_wrapper,    ///< Histogram counter data having logical dimensions <tt>CounterT[NUM_ACTIVE_CHANNELS][num_bins.array[CHANNEL]]</tt>
-    GridQueue<int>                                  tile_queue)                     ///< Drain queue descriptor for dynamically mapping tile data onto thread blocks
-{
-    if ((threadIdx.x == 0) && (blockIdx.x == 0))
-        tile_queue.ResetDrain();
-
-    int output_bin = (blockIdx.x * blockDim.x) + threadIdx.x;
-
-    #pragma unroll
-    for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
-    {
-        if (output_bin < num_output_bins_wrapper.array[CHANNEL])
-            d_output_histograms_wrapper.array[CHANNEL][output_bin] = 0;
-    }
-}
-
-
-/**
- * Histogram privatized sweep kernel entry point (multi-block).  Computes privatized histograms, one per thread block.
- */
-template <
-    typename                                            AgentHistogramPolicyT,     ///< Parameterized AgentHistogramPolicy tuning policy type
-    int                                                 PRIVATIZED_SMEM_BINS,           ///< Maximum number of histogram bins per channel (e.g., up to 256)
-    int                                                 NUM_CHANNELS,                   ///< Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed)
-    int                                                 NUM_ACTIVE_CHANNELS,            ///< Number of channels actively being histogrammed
-    typename                                            SampleIteratorT,                ///< The input iterator type. \iterator.
-    typename                                            CounterT,                       ///< Integer type for counting sample occurrences per histogram bin
-    typename                                            PrivatizedDecodeOpT,            ///< The transform operator type for determining privatized counter indices from samples, one for each channel
-    typename                                            OutputDecodeOpT,                ///< The transform operator type for determining output bin-ids from privatized counter indices, one for each channel
-    typename                                            OffsetT>                        ///< Signed integer type for global offsets
-__launch_bounds__ (int(AgentHistogramPolicyT::BLOCK_THREADS))
-__global__ void DeviceHistogramSweepKernel(
-    SampleIteratorT                                         d_samples,                          ///< Input data to reduce
-    ArrayWrapper<int, NUM_ACTIVE_CHANNELS>                  num_output_bins_wrapper,            ///< The number bins per final output histogram
-    ArrayWrapper<int, NUM_ACTIVE_CHANNELS>                  num_privatized_bins_wrapper,        ///< The number bins per privatized histogram
-    ArrayWrapper<CounterT*, NUM_ACTIVE_CHANNELS>            d_output_histograms_wrapper,        ///< Reference to final output histograms
-    ArrayWrapper<CounterT*, NUM_ACTIVE_CHANNELS>            d_privatized_histograms_wrapper,    ///< Reference to privatized histograms
-    ArrayWrapper<OutputDecodeOpT, NUM_ACTIVE_CHANNELS>      output_decode_op_wrapper,           ///< The transform operator for determining output bin-ids from privatized counter indices, one for each channel
-    ArrayWrapper<PrivatizedDecodeOpT, NUM_ACTIVE_CHANNELS>  privatized_decode_op_wrapper,       ///< The transform operator for determining privatized counter indices from samples, one for each channel
-    OffsetT                                                 num_row_pixels,                     ///< The number of multi-channel pixels per row in the region of interest
-    OffsetT                                                 num_rows,                           ///< The number of rows in the region of interest
-    OffsetT                                                 row_stride_samples,                 ///< The number of samples between starts of consecutive rows in the region of interest
-    int                                                     tiles_per_row,                      ///< Number of image tiles per row
-    GridQueue<int>                                          tile_queue)                         ///< Drain queue descriptor for dynamically mapping tile data onto thread blocks
-{
-    // Thread block type for compositing input tiles
-    typedef AgentHistogram<
-            AgentHistogramPolicyT,
-            PRIVATIZED_SMEM_BINS,
-            NUM_CHANNELS,
-            NUM_ACTIVE_CHANNELS,
-            SampleIteratorT,
-            CounterT,
-            PrivatizedDecodeOpT,
-            OutputDecodeOpT,
-            OffsetT>
-        AgentHistogramT;
-
-    // Shared memory for AgentHistogram
-    __shared__ typename AgentHistogramT::TempStorage temp_storage;
-
-    AgentHistogramT agent(
-        temp_storage,
-        d_samples,
-        num_output_bins_wrapper.array,
-        num_privatized_bins_wrapper.array,
-        d_output_histograms_wrapper.array,
-        d_privatized_histograms_wrapper.array,
-        output_decode_op_wrapper.array,
-        privatized_decode_op_wrapper.array);
-
-    // Initialize counters
-    agent.InitBinCounters();
-
-    // Consume input tiles
-    agent.ConsumeTiles(
-        num_row_pixels,
-        num_rows,
-        row_stride_samples,
-        tiles_per_row,
-        tile_queue);
-
-    // Store output to global (if necessary)
-    agent.StoreOutput();
-
-}
-
-
-
-
-
-
-/******************************************************************************
- * Dispatch
- ******************************************************************************/
-
-/**
- * Utility class for dispatching the appropriately-tuned kernels for DeviceHistogram
- */
-template <
-    int         NUM_CHANNELS,               ///< Number of channels interleaved in the input data (may be greater than the number of channels being actively histogrammed)
-    int         NUM_ACTIVE_CHANNELS,        ///< Number of channels actively being histogrammed
-    typename    SampleIteratorT,            ///< Random-access input iterator type for reading input items \iterator
-    typename    CounterT,                   ///< Integer type for counting sample occurrences per histogram bin
-    typename    LevelT,                     ///< Type for specifying bin level boundaries
-    typename    OffsetT>                    ///< Signed integer type for global offsets
-struct DipatchHistogram
-{
-    //---------------------------------------------------------------------
-    // Types and constants
-    //---------------------------------------------------------------------
-
-    /// The sample value type of the input iterator
-    typedef typename std::iterator_traits<SampleIteratorT>::value_type SampleT;
-
-    enum
-    {
-        // Maximum number of bins per channel for which we will use a privatized smem strategy
-        MAX_PRIVATIZED_SMEM_BINS = 256
-    };
-
-
-    //---------------------------------------------------------------------
-    // Transform functors for converting samples to bin-ids
-    //---------------------------------------------------------------------
-
-    // Searches for bin given a list of bin-boundary levels
-    template <typename LevelIteratorT>
-    struct SearchTransform
-    {
-        LevelIteratorT  d_levels;                   // Pointer to levels array
-        int             num_output_levels;          // Number of levels in array
-
-        // Initializer
-        __host__ __device__ __forceinline__ void Init(
-            LevelIteratorT  d_levels,               // Pointer to levels array
-            int             num_output_levels)      // Number of levels in array
-        {
-            this->d_levels          = d_levels;
-            this->num_output_levels = num_output_levels;
-        }
-
-        // Method for converting samples to bin-ids
-        template <CacheLoadModifier LOAD_MODIFIER, typename _SampleT>
-        __host__ __device__ __forceinline__ void BinSelect(_SampleT sample, int &bin, bool valid)
-        {
-            /// Level iterator wrapper type
-            typedef typename If<IsPointer<LevelIteratorT>::VALUE,
-                    CacheModifiedInputIterator<LOAD_MODIFIER, LevelT, OffsetT>,     // Wrap the native input pointer with CacheModifiedInputIterator
-                    LevelIteratorT>::Type                                           // Directly use the supplied input iterator type
-                WrappedLevelIteratorT;
-
-            WrappedLevelIteratorT wrapped_levels(d_levels);
-
-            int num_bins = num_output_levels - 1;
-            if (valid)
-            {
-                bin = UpperBound(wrapped_levels, num_output_levels, (LevelT) sample) - 1;
-                if (bin >= num_bins)
-                    bin = -1;
-            }
-        }
-    };
-
-
-    // Scales samples to evenly-spaced bins
-    struct ScaleTransform
-    {
-        int    num_bins;    // Number of levels in array
-        LevelT max;         // Max sample level (exclusive)
-        LevelT min;         // Min sample level (inclusive)
-        LevelT scale;       // Bin scaling factor
-
-        // Initializer
-        template <typename _LevelT>
-        __host__ __device__ __forceinline__ void Init(
-            int     num_output_levels,  // Number of levels in array
-            _LevelT max,                // Max sample level (exclusive)
-            _LevelT min,                // Min sample level (inclusive)
-            _LevelT scale)              // Bin scaling factor
-        {
-            this->num_bins = num_output_levels - 1;
-            this->max = max;
-            this->min = min;
-            this->scale = scale;
-        }
-
-        // Initializer (float specialization)
-        __host__ __device__ __forceinline__ void Init(
-            int    num_output_levels,   // Number of levels in array
-            float   max,                // Max sample level (exclusive)
-            float   min,                // Min sample level (inclusive)
-            float   scale)              // Bin scaling factor
-        {
-            this->num_bins = num_output_levels - 1;
-            this->max = max;
-            this->min = min;
-            this->scale = float(1.0) / scale;
-        }
-
-        // Initializer (double specialization)
-        __host__ __device__ __forceinline__ void Init(
-            int    num_output_levels,   // Number of levels in array
-            double max,                 // Max sample level (exclusive)
-            double min,                 // Min sample level (inclusive)
-            double scale)               // Bin scaling factor
-        {
-            this->num_bins = num_output_levels - 1;
-            this->max = max;
-            this->min = min;
-            this->scale = double(1.0) / scale;
-        }
-
-        // Method for converting samples to bin-ids
-        template <CacheLoadModifier LOAD_MODIFIER, typename _SampleT>
-        __host__ __device__ __forceinline__ void BinSelect(_SampleT sample, int &bin, bool valid)
-        {
-            LevelT level_sample = (LevelT) sample;
-
-            if (valid && (level_sample >= min) && (level_sample < max))
-                bin = (int) ((level_sample - min) / scale);
-        }
-
-        // Method for converting samples to bin-ids (float specialization)
-        template <CacheLoadModifier LOAD_MODIFIER>
-        __host__ __device__ __forceinline__ void BinSelect(float sample, int &bin, bool valid)
-        {
-            LevelT level_sample = (LevelT) sample;
-
-            if (valid && (level_sample >= min) && (level_sample < max))
-                bin = (int) ((level_sample - min) * scale);
-        }
-
-        // Method for converting samples to bin-ids (double specialization)
-        template <CacheLoadModifier LOAD_MODIFIER>
-        __host__ __device__ __forceinline__ void BinSelect(double sample, int &bin, bool valid)
-        {
-            LevelT level_sample = (LevelT) sample;
-
-            if (valid && (level_sample >= min) && (level_sample < max))
-                bin = (int) ((level_sample - min) * scale);
-        }
-    };
-
-
-    // Pass-through bin transform operator
-    struct PassThruTransform
-    {
-        // Method for converting samples to bin-ids
-        template <CacheLoadModifier LOAD_MODIFIER, typename _SampleT>
-        __host__ __device__ __forceinline__ void BinSelect(_SampleT sample, int &bin, bool valid)
-        {
-            if (valid)
-                bin = (int) sample;
-        }
-    };
-
-
-
-    //---------------------------------------------------------------------
-    // Tuning policies
-    //---------------------------------------------------------------------
-
-    template <int NOMINAL_ITEMS_PER_THREAD>
-    struct TScale
-    {
-        enum
-        {
-            V_SCALE = (sizeof(SampleT) + sizeof(int) - 1) / sizeof(int),
-            VALUE   = CUB_MAX((NOMINAL_ITEMS_PER_THREAD / NUM_ACTIVE_CHANNELS / V_SCALE), 1)
-        };
-    };
-
-
-    /// SM11
-    struct Policy110
-    {
-        // HistogramSweepPolicy
-        typedef AgentHistogramPolicy<
-                512,
-                (NUM_CHANNELS == 1) ? 8 : 2,
-                BLOCK_LOAD_DIRECT,
-                LOAD_DEFAULT,
-                true,
-                GMEM,
-                false>
-            HistogramSweepPolicy;
-    };
-
-    /// SM20
-    struct Policy200
-    {
-        // HistogramSweepPolicy
-        typedef AgentHistogramPolicy<
-                (NUM_CHANNELS == 1) ? 256 : 128,
-                (NUM_CHANNELS == 1) ? 8 : 3,
-                (NUM_CHANNELS == 1) ? BLOCK_LOAD_DIRECT : BLOCK_LOAD_WARP_TRANSPOSE,
-                LOAD_DEFAULT,
-                true,
-                SMEM,
-                false>
-            HistogramSweepPolicy;
-    };
-
-    /// SM30
-    struct Policy300
-    {
-        // HistogramSweepPolicy
-        typedef AgentHistogramPolicy<
-                512,
-                (NUM_CHANNELS == 1) ? 8 : 2,
-                BLOCK_LOAD_DIRECT,
-                LOAD_DEFAULT,
-                true,
-                GMEM,
-                false>
-            HistogramSweepPolicy;
-    };
-
-    /// SM35
-    struct Policy350
-    {
-        // HistogramSweepPolicy
-        typedef AgentHistogramPolicy<
-                128,
-                TScale<8>::VALUE,
-                BLOCK_LOAD_DIRECT,
-                LOAD_LDG,
-                true,
-                BLEND,
-                true>
-            HistogramSweepPolicy;
-    };
-
-    /// SM50
-    struct Policy500
-    {
-        // HistogramSweepPolicy
-        typedef AgentHistogramPolicy<
-                384,
-                TScale<16>::VALUE,
-                BLOCK_LOAD_DIRECT,
-                LOAD_LDG,
-                true,
-                SMEM,
-                false>
-            HistogramSweepPolicy;
-    };
-
-
-
-    //---------------------------------------------------------------------
-    // Tuning policies of current PTX compiler pass
-    //---------------------------------------------------------------------
-
-#if (CUB_PTX_ARCH >= 500)
-    typedef Policy500 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 350)
-    typedef Policy350 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 300)
-    typedef Policy300 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 200)
-    typedef Policy200 PtxPolicy;
-
-#else
-    typedef Policy110 PtxPolicy;
-
-#endif
-
-    // "Opaque" policies (whose parameterizations aren't reflected in the type signature)
-    struct PtxHistogramSweepPolicy : PtxPolicy::HistogramSweepPolicy {};
-
-
-    //---------------------------------------------------------------------
-    // Utilities
-    //---------------------------------------------------------------------
-
-    /**
-     * Initialize kernel dispatch configurations with the policies corresponding to the PTX assembly we will use
-     */
-    template <typename KernelConfig>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t InitConfigs(
-        int             ptx_version,
-        KernelConfig    &histogram_sweep_config)
-    {
-    #if (CUB_PTX_ARCH > 0)
-
-        // We're on the device, so initialize the kernel dispatch configurations with the current PTX policy
-        return histogram_sweep_config.template Init<PtxHistogramSweepPolicy>();
-
-    #else
-
-        // We're on the host, so lookup and initialize the kernel dispatch configurations with the policies that match the device's PTX version
-        if (ptx_version >= 500)
-        {
-            return histogram_sweep_config.template Init<typename Policy500::HistogramSweepPolicy>();
-        }
-        else if (ptx_version >= 350)
-        {
-            return histogram_sweep_config.template Init<typename Policy350::HistogramSweepPolicy>();
-        }
-        else if (ptx_version >= 300)
-        {
-            return histogram_sweep_config.template Init<typename Policy300::HistogramSweepPolicy>();
-        }
-        else if (ptx_version >= 200)
-        {
-            return histogram_sweep_config.template Init<typename Policy200::HistogramSweepPolicy>();
-        }
-        else if (ptx_version >= 110)
-        {
-            return histogram_sweep_config.template Init<typename Policy110::HistogramSweepPolicy>();
-        }
-        else
-        {
-            // No global atomic support
-            return cudaErrorNotSupported;
-        }
-
-    #endif
-    }
-
-
-    /**
-     * Kernel kernel dispatch configuration
-     */
-    struct KernelConfig
-    {
-        int                             block_threads;
-        int                             pixels_per_thread;
-
-        template <typename BlockPolicy>
-        CUB_RUNTIME_FUNCTION __forceinline__
-        cudaError_t Init()
-        {
-            block_threads               = BlockPolicy::BLOCK_THREADS;
-            pixels_per_thread           = BlockPolicy::PIXELS_PER_THREAD;
-
-            return cudaSuccess;
-        }
-    };
-
-
-    //---------------------------------------------------------------------
-    // Dispatch entrypoints
-    //---------------------------------------------------------------------
-
-    /**
-     * Privatization-based dispatch routine
-     */
-    template <
-        typename                            PrivatizedDecodeOpT,                            ///< The transform operator type for determining privatized counter indices from samples, one for each channel
-        typename                            OutputDecodeOpT,                                ///< The transform operator type for determining output bin-ids from privatized counter indices, one for each channel
-        typename                            DeviceHistogramInitKernelT,                     ///< Function type of cub::DeviceHistogramInitKernel
-        typename                            DeviceHistogramSweepKernelT>                    ///< Function type of cub::DeviceHistogramSweepKernel
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t PrivatizedDispatch(
-        void*                               d_temp_storage,                                 ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&                             temp_storage_bytes,                             ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        SampleIteratorT                     d_samples,                                      ///< [in] The pointer to the input sequence of sample items. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four RGBA 8-bit samples).
-        CounterT*                           d_output_histograms[NUM_ACTIVE_CHANNELS],       ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histograms[i]</tt> should be <tt>num_output_levels[i]</tt> - 1.
-        int                                 num_privatized_levels[NUM_ACTIVE_CHANNELS],     ///< [in] The number of bin level boundaries for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_output_levels[i]</tt> - 1.
-        PrivatizedDecodeOpT                 privatized_decode_op[NUM_ACTIVE_CHANNELS],      ///< [in] Transform operators for determining bin-ids from samples, one for each channel
-        int                                 num_output_levels[NUM_ACTIVE_CHANNELS],         ///< [in] The number of bin level boundaries for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_output_levels[i]</tt> - 1.
-        OutputDecodeOpT                     output_decode_op[NUM_ACTIVE_CHANNELS],          ///< [in] Transform operators for determining bin-ids from samples, one for each channel
-        int                                 max_num_output_bins,                            ///< [in] Maximum number of output bins in any channel
-        OffsetT                             num_row_pixels,                                 ///< [in] The number of multi-channel pixels per row in the region of interest
-        OffsetT                             num_rows,                                       ///< [in] The number of rows in the region of interest
-        OffsetT                             row_stride_samples,                             ///< [in] The number of samples between starts of consecutive rows in the region of interest
-        DeviceHistogramInitKernelT          histogram_init_kernel,                          ///< [in] Kernel function pointer to parameterization of cub::DeviceHistogramInitKernel
-        DeviceHistogramSweepKernelT         histogram_sweep_kernel,                         ///< [in] Kernel function pointer to parameterization of cub::DeviceHistogramSweepKernel
-        KernelConfig                        histogram_sweep_config,                         ///< [in] Dispatch parameters that match the policy that \p histogram_sweep_kernel was compiled for
-        cudaStream_t                        stream,                                         ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                                debug_synchronous)                              ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-    #ifndef CUB_RUNTIME_ENABLED
-
-        // Kernel launch not supported from this device
-        return CubDebug(cudaErrorNotSupported);
-
-    #else
-
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Get device ordinal
-            int device_ordinal;
-            if (CubDebug(error = cudaGetDevice(&device_ordinal))) break;
-
-            // Get SM count
-            int sm_count;
-            if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break;
-
-            // Get SM occupancy for histogram_sweep_kernel
-            int histogram_sweep_sm_occupancy;
-            if (CubDebug(error = MaxSmOccupancy(
-                histogram_sweep_sm_occupancy,
-                histogram_sweep_kernel,
-                histogram_sweep_config.block_threads))) break;
-
-            // Get device occupancy for histogram_sweep_kernel
-            int histogram_sweep_occupancy = histogram_sweep_sm_occupancy * sm_count;
-
-            if (num_row_pixels * NUM_CHANNELS == row_stride_samples)
-            {
-                // Treat as a single linear array of samples
-                num_row_pixels      *= num_rows;
-                num_rows            = 1;
-                row_stride_samples  = num_row_pixels * NUM_CHANNELS;
-            }
-
-            // Get grid dimensions, trying to keep total blocks ~histogram_sweep_occupancy
-            int pixels_per_tile     = histogram_sweep_config.block_threads * histogram_sweep_config.pixels_per_thread;
-            int tiles_per_row       = int(num_row_pixels + pixels_per_tile - 1) / pixels_per_tile;
-            int blocks_per_row      = CUB_MIN(histogram_sweep_occupancy, tiles_per_row);
-            int blocks_per_col      = (blocks_per_row > 0) ?
-                                        int(CUB_MIN(histogram_sweep_occupancy / blocks_per_row, num_rows)) :
-                                        0;
-            int num_thread_blocks   = blocks_per_row * blocks_per_col;
-
-            dim3 sweep_grid_dims;
-            sweep_grid_dims.x = (unsigned int) blocks_per_row;
-            sweep_grid_dims.y = (unsigned int) blocks_per_col;
-            sweep_grid_dims.z = 1;
-
-            // Temporary storage allocation requirements
-            const int   NUM_ALLOCATIONS = NUM_ACTIVE_CHANNELS + 1;
-            void*       allocations[NUM_ALLOCATIONS];
-            size_t      allocation_sizes[NUM_ALLOCATIONS];
-
-            for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
-                allocation_sizes[CHANNEL] = size_t(num_thread_blocks) * (num_privatized_levels[CHANNEL] - 1) * sizeof(CounterT);
-
-            allocation_sizes[NUM_ALLOCATIONS - 1] = GridQueue<int>::AllocationSize();
-
-            // Alias the temporary allocations from the single storage blob (or compute the necessary size of the blob)
-            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
-            if (d_temp_storage == NULL)
-            {
-                // Return if the caller is simply requesting the size of the storage allocation
-                break;
-            }
-
-            // Construct the grid queue descriptor
-            GridQueue<int> tile_queue(allocations[NUM_ALLOCATIONS - 1]);
-
-            // Setup array wrapper for histogram channel output (because we can't pass static arrays as kernel parameters)
-            ArrayWrapper<CounterT*, NUM_ACTIVE_CHANNELS> d_output_histograms_wrapper;
-            for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
-                d_output_histograms_wrapper.array[CHANNEL] = d_output_histograms[CHANNEL];
-
-            // Setup array wrapper for privatized per-block histogram channel output (because we can't pass static arrays as kernel parameters)
-            ArrayWrapper<CounterT*, NUM_ACTIVE_CHANNELS> d_privatized_histograms_wrapper;
-            for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
-                d_privatized_histograms_wrapper.array[CHANNEL] = (CounterT*) allocations[CHANNEL];
-
-            // Setup array wrapper for sweep bin transforms (because we can't pass static arrays as kernel parameters)
-            ArrayWrapper<PrivatizedDecodeOpT, NUM_ACTIVE_CHANNELS> privatized_decode_op_wrapper;
-            for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
-                privatized_decode_op_wrapper.array[CHANNEL] = privatized_decode_op[CHANNEL];
-
-            // Setup array wrapper for aggregation bin transforms (because we can't pass static arrays as kernel parameters)
-            ArrayWrapper<OutputDecodeOpT, NUM_ACTIVE_CHANNELS> output_decode_op_wrapper;
-            for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
-                output_decode_op_wrapper.array[CHANNEL] = output_decode_op[CHANNEL];
-
-            // Setup array wrapper for num privatized bins (because we can't pass static arrays as kernel parameters)
-            ArrayWrapper<int, NUM_ACTIVE_CHANNELS> num_privatized_bins_wrapper;
-            for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
-                num_privatized_bins_wrapper.array[CHANNEL] = num_privatized_levels[CHANNEL] - 1;
-
-            // Setup array wrapper for num output bins (because we can't pass static arrays as kernel parameters)
-            ArrayWrapper<int, NUM_ACTIVE_CHANNELS> num_output_bins_wrapper;
-            for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
-                num_output_bins_wrapper.array[CHANNEL] = num_output_levels[CHANNEL] - 1;
-
-            int histogram_init_block_threads    = 256;
-            int histogram_init_grid_dims        = (max_num_output_bins + histogram_init_block_threads - 1) / histogram_init_block_threads;
-
-            // Log DeviceHistogramInitKernel configuration
-            if (debug_synchronous) _CubLog("Invoking DeviceHistogramInitKernel<<<%d, %d, 0, %lld>>>()\n",
-                histogram_init_grid_dims, histogram_init_block_threads, (long long) stream);
-
-            // Invoke histogram_init_kernel
-            histogram_init_kernel<<<histogram_init_grid_dims, histogram_init_block_threads, 0, stream>>>(
-                num_output_bins_wrapper,
-                d_output_histograms_wrapper,
-                tile_queue);
-
-            // Return if empty problem
-            if ((blocks_per_row == 0) || (blocks_per_col == 0))
-                break;
-
-            // Log histogram_sweep_kernel configuration
-            if (debug_synchronous) _CubLog("Invoking histogram_sweep_kernel<<<{%d, %d, %d}, %d, 0, %lld>>>(), %d pixels per thread, %d SM occupancy\n",
-                sweep_grid_dims.x, sweep_grid_dims.y, sweep_grid_dims.z,
-                histogram_sweep_config.block_threads, (long long) stream, histogram_sweep_config.pixels_per_thread, histogram_sweep_sm_occupancy);
-
-            // Invoke histogram_sweep_kernel
-            histogram_sweep_kernel<<<sweep_grid_dims, histogram_sweep_config.block_threads, 0, stream>>>(
-                d_samples,
-                num_output_bins_wrapper,
-                num_privatized_bins_wrapper,
-                d_output_histograms_wrapper,
-                d_privatized_histograms_wrapper,
-                output_decode_op_wrapper,
-                privatized_decode_op_wrapper,
-                num_row_pixels,
-                num_rows,
-                row_stride_samples,
-                tiles_per_row,
-                tile_queue);
-
-            // Check for failure to launch
-            if (CubDebug(error = cudaPeekAtLastError())) break;
-
-            // Sync the stream if specified to flush runtime errors
-            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-
-        }
-        while (0);
-
-        return error;
-
-    #endif // CUB_RUNTIME_ENABLED
-    }
-
-
-
-    /**
-     * Dispatch routine for HistogramRange, specialized for sample types larger than 8bit
-     */
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t DispatchRange(
-        void*               d_temp_storage,                                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&             temp_storage_bytes,                            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        SampleIteratorT     d_samples,                                  ///< [in] The pointer to the multi-channel input sequence of data samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four RGBA 8-bit samples).
-        CounterT*           d_output_histograms[NUM_ACTIVE_CHANNELS],      ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histograms[i]</tt> should be <tt>num_output_levels[i]</tt> - 1.
-        int                 num_output_levels[NUM_ACTIVE_CHANNELS],     ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_output_levels[i]</tt> - 1.
-        LevelT              *d_levels[NUM_ACTIVE_CHANNELS],             ///< [in] The pointers to the arrays of boundaries (levels), one for each active channel.  Bin ranges are defined by consecutive boundary pairings: lower sample value boundaries are inclusive and upper sample value boundaries are exclusive.
-        OffsetT             num_row_pixels,                             ///< [in] The number of multi-channel pixels per row in the region of interest
-        OffsetT             num_rows,                                   ///< [in] The number of rows in the region of interest
-        OffsetT             row_stride_samples,                         ///< [in] The number of samples between starts of consecutive rows in the region of interest
-        cudaStream_t        stream,                                     ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous,                          ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-        Int2Type<false>     is_byte_sample)                             ///< [in] Marker type indicating whether or not SampleT is a 8b type
-    {
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Get PTX version
-            int ptx_version;
-    #if (CUB_PTX_ARCH == 0)
-            if (CubDebug(error = PtxVersion(ptx_version))) break;
-    #else
-            ptx_version = CUB_PTX_ARCH;
-    #endif
-
-            // Get kernel dispatch configurations
-            KernelConfig histogram_sweep_config;
-            if (CubDebug(error = InitConfigs(ptx_version, histogram_sweep_config)))
-                break;
-
-            // Use the search transform op for converting samples to privatized bins
-            typedef SearchTransform<LevelT*> PrivatizedDecodeOpT;
-
-            // Use the pass-thru transform op for converting privatized bins to output bins
-            typedef PassThruTransform OutputDecodeOpT;
-
-            PrivatizedDecodeOpT     privatized_decode_op[NUM_ACTIVE_CHANNELS];
-            OutputDecodeOpT         output_decode_op[NUM_ACTIVE_CHANNELS];
-            int                     max_levels = num_output_levels[0];
-
-            for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
-            {
-                privatized_decode_op[channel].Init(d_levels[channel], num_output_levels[channel]);
-                if (num_output_levels[channel] > max_levels)
-                    max_levels = num_output_levels[channel];
-            }
-            int max_num_output_bins = max_levels - 1;
-
-            // Dispatch
-            if (max_num_output_bins > MAX_PRIVATIZED_SMEM_BINS)
-            {
-                // Too many bins to keep in shared memory.
-                const int PRIVATIZED_SMEM_BINS = 0;
-
-                if (CubDebug(error = PrivatizedDispatch(
-                    d_temp_storage,
-                    temp_storage_bytes,
-                    d_samples,
-                    d_output_histograms,
-                    num_output_levels,
-                    privatized_decode_op,
-                    num_output_levels,
-                    output_decode_op,
-                    max_num_output_bins,
-                    num_row_pixels,
-                    num_rows,
-                    row_stride_samples,
-                    DeviceHistogramInitKernel<NUM_ACTIVE_CHANNELS, CounterT, OffsetT>,
-                    DeviceHistogramSweepKernel<PtxHistogramSweepPolicy, PRIVATIZED_SMEM_BINS, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, PrivatizedDecodeOpT, OutputDecodeOpT, OffsetT>,
-                    histogram_sweep_config,
-                    stream,
-                    debug_synchronous))) break;
-            }
-            else
-            {
-                // Dispatch shared-privatized approach
-                const int PRIVATIZED_SMEM_BINS = MAX_PRIVATIZED_SMEM_BINS;
-
-                if (CubDebug(error = PrivatizedDispatch(
-                    d_temp_storage,
-                    temp_storage_bytes,
-                    d_samples,
-                    d_output_histograms,
-                    num_output_levels,
-                    privatized_decode_op,
-                    num_output_levels,
-                    output_decode_op,
-                    max_num_output_bins,
-                    num_row_pixels,
-                    num_rows,
-                    row_stride_samples,
-                    DeviceHistogramInitKernel<NUM_ACTIVE_CHANNELS, CounterT, OffsetT>,
-                    DeviceHistogramSweepKernel<PtxHistogramSweepPolicy, PRIVATIZED_SMEM_BINS, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, PrivatizedDecodeOpT, OutputDecodeOpT, OffsetT>,
-                    histogram_sweep_config,
-                    stream,
-                    debug_synchronous))) break;
-            }
-
-        } while (0);
-
-        return error;
-    }
-
-
-    /**
-     * Dispatch routine for HistogramRange, specialized for 8-bit sample types (computes 256-bin privatized histograms and then reduces to user-specified levels)
-     */
-    CUB_RUNTIME_FUNCTION
-    static cudaError_t DispatchRange(
-        void*               d_temp_storage,                             ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&             temp_storage_bytes,                         ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        SampleIteratorT     d_samples,                                  ///< [in] The pointer to the multi-channel input sequence of data samples. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four RGBA 8-bit samples).
-        CounterT*           d_output_histograms[NUM_ACTIVE_CHANNELS],   ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histograms[i]</tt> should be <tt>num_output_levels[i]</tt> - 1.
-        int                 num_output_levels[NUM_ACTIVE_CHANNELS],     ///< [in] The number of boundaries (levels) for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_output_levels[i]</tt> - 1.
-        LevelT              *d_levels[NUM_ACTIVE_CHANNELS],             ///< [in] The pointers to the arrays of boundaries (levels), one for each active channel.  Bin ranges are defined by consecutive boundary pairings: lower sample value boundaries are inclusive and upper sample value boundaries are exclusive.
-        OffsetT             num_row_pixels,                             ///< [in] The number of multi-channel pixels per row in the region of interest
-        OffsetT             num_rows,                                   ///< [in] The number of rows in the region of interest
-        OffsetT             row_stride_samples,                         ///< [in] The number of samples between starts of consecutive rows in the region of interest
-        cudaStream_t        stream,                                     ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous,                          ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-        Int2Type<true>      is_byte_sample)                             ///< [in] Marker type indicating whether or not SampleT is a 8b type
-    {
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Get PTX version
-            int ptx_version;
-    #if (CUB_PTX_ARCH == 0)
-            if (CubDebug(error = PtxVersion(ptx_version))) break;
-    #else
-            ptx_version = CUB_PTX_ARCH;
-    #endif
-
-            // Get kernel dispatch configurations
-            KernelConfig histogram_sweep_config;
-            if (CubDebug(error = InitConfigs(ptx_version, histogram_sweep_config)))
-                break;
-
-            // Use the pass-thru transform op for converting samples to privatized bins
-            typedef PassThruTransform PrivatizedDecodeOpT;
-
-            // Use the search transform op for converting privatized bins to output bins
-            typedef SearchTransform<LevelT*> OutputDecodeOpT;
-
-            int                         num_privatized_levels[NUM_ACTIVE_CHANNELS];
-            PrivatizedDecodeOpT         privatized_decode_op[NUM_ACTIVE_CHANNELS];
-            OutputDecodeOpT             output_decode_op[NUM_ACTIVE_CHANNELS];
-            int                         max_levels = num_output_levels[0];              // Maximum number of levels in any channel
-
-            for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
-            {
-                num_privatized_levels[channel] = 257;
-                output_decode_op[channel].Init(d_levels[channel], num_output_levels[channel]);
-
-                if (num_output_levels[channel] > max_levels)
-                    max_levels = num_output_levels[channel];
-            }
-            int max_num_output_bins = max_levels - 1;
-
-            const int PRIVATIZED_SMEM_BINS = 256;
-
-            if (CubDebug(error = PrivatizedDispatch(
-                d_temp_storage,
-                temp_storage_bytes,
-                d_samples,
-                d_output_histograms,
-                num_privatized_levels,
-                privatized_decode_op,
-                num_output_levels,
-                output_decode_op,
-                max_num_output_bins,
-                num_row_pixels,
-                num_rows,
-                row_stride_samples,
-                DeviceHistogramInitKernel<NUM_ACTIVE_CHANNELS, CounterT, OffsetT>,
-                DeviceHistogramSweepKernel<PtxHistogramSweepPolicy, PRIVATIZED_SMEM_BINS, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, PrivatizedDecodeOpT, OutputDecodeOpT, OffsetT>,
-                histogram_sweep_config,
-                stream,
-                debug_synchronous))) break;
-
-        } while (0);
-
-        return error;
-    }
-
-
-    /**
-     * Dispatch routine for HistogramEven, specialized for sample types larger than 8-bit
-     */
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t DispatchEven(
-        void*               d_temp_storage,                            ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&             temp_storage_bytes,                        ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        SampleIteratorT     d_samples,                                  ///< [in] The pointer to the input sequence of sample items. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four RGBA 8-bit samples).
-        CounterT*           d_output_histograms[NUM_ACTIVE_CHANNELS],  ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histograms[i]</tt> should be <tt>num_output_levels[i]</tt> - 1.
-        int                 num_output_levels[NUM_ACTIVE_CHANNELS],     ///< [in] The number of bin level boundaries for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_output_levels[i]</tt> - 1.
-        LevelT              lower_level[NUM_ACTIVE_CHANNELS],           ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin in each active channel.
-        LevelT              upper_level[NUM_ACTIVE_CHANNELS],           ///< [in] The upper sample value bound (exclusive) for the highest histogram bin in each active channel.
-        OffsetT             num_row_pixels,                             ///< [in] The number of multi-channel pixels per row in the region of interest
-        OffsetT             num_rows,                                   ///< [in] The number of rows in the region of interest
-        OffsetT             row_stride_samples,                         ///< [in] The number of samples between starts of consecutive rows in the region of interest
-        cudaStream_t        stream,                                     ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous,                          ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-        Int2Type<false>     is_byte_sample)                             ///< [in] Marker type indicating whether or not SampleT is a 8b type
-    {
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Get PTX version
-            int ptx_version;
-    #if (CUB_PTX_ARCH == 0)
-            if (CubDebug(error = PtxVersion(ptx_version))) break;
-    #else
-            ptx_version = CUB_PTX_ARCH;
-    #endif
-
-            // Get kernel dispatch configurations
-            KernelConfig histogram_sweep_config;
-            if (CubDebug(error = InitConfigs(ptx_version, histogram_sweep_config)))
-                break;
-
-            // Use the scale transform op for converting samples to privatized bins
-            typedef ScaleTransform PrivatizedDecodeOpT;
-
-            // Use the pass-thru transform op for converting privatized bins to output bins
-            typedef PassThruTransform OutputDecodeOpT;
-
-            PrivatizedDecodeOpT         privatized_decode_op[NUM_ACTIVE_CHANNELS];
-            OutputDecodeOpT             output_decode_op[NUM_ACTIVE_CHANNELS];
-            int                         max_levels = num_output_levels[0];
-
-            for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
-            {
-                int     bins    = num_output_levels[channel] - 1;
-                LevelT  scale   = (upper_level[channel] - lower_level[channel]) / bins;
-
-                privatized_decode_op[channel].Init(num_output_levels[channel], upper_level[channel], lower_level[channel], scale);
-
-                if (num_output_levels[channel] > max_levels)
-                    max_levels = num_output_levels[channel];
-            }
-            int max_num_output_bins = max_levels - 1;
-
-            if (max_num_output_bins > MAX_PRIVATIZED_SMEM_BINS)
-            {
-                // Dispatch shared-privatized approach
-                const int PRIVATIZED_SMEM_BINS = 0;
-
-                if (CubDebug(error = PrivatizedDispatch(
-                    d_temp_storage,
-                    temp_storage_bytes,
-                    d_samples,
-                    d_output_histograms,
-                    num_output_levels,
-                    privatized_decode_op,
-                    num_output_levels,
-                    output_decode_op,
-                    max_num_output_bins,
-                    num_row_pixels,
-                    num_rows,
-                    row_stride_samples,
-                    DeviceHistogramInitKernel<NUM_ACTIVE_CHANNELS, CounterT, OffsetT>,
-                    DeviceHistogramSweepKernel<PtxHistogramSweepPolicy, PRIVATIZED_SMEM_BINS, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, PrivatizedDecodeOpT, OutputDecodeOpT, OffsetT>,
-                    histogram_sweep_config,
-                    stream,
-                    debug_synchronous))) break;
-            }
-            else
-            {
-                // Dispatch shared-privatized approach
-                const int PRIVATIZED_SMEM_BINS = MAX_PRIVATIZED_SMEM_BINS;
-
-                if (CubDebug(error = PrivatizedDispatch(
-                    d_temp_storage,
-                    temp_storage_bytes,
-                    d_samples,
-                    d_output_histograms,
-                    num_output_levels,
-                    privatized_decode_op,
-                    num_output_levels,
-                    output_decode_op,
-                    max_num_output_bins,
-                    num_row_pixels,
-                    num_rows,
-                    row_stride_samples,
-                    DeviceHistogramInitKernel<NUM_ACTIVE_CHANNELS, CounterT, OffsetT>,
-                    DeviceHistogramSweepKernel<PtxHistogramSweepPolicy, PRIVATIZED_SMEM_BINS, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, PrivatizedDecodeOpT, OutputDecodeOpT, OffsetT>,
-                    histogram_sweep_config,
-                    stream,
-                    debug_synchronous))) break;
-            }
-        }
-        while (0);
-
-        return error;
-    }
-
-
-    /**
-     * Dispatch routine for HistogramEven, specialized for 8-bit sample types (computes 256-bin privatized histograms and then reduces to user-specified levels)
-     */
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t DispatchEven(
-        void*               d_temp_storage,                            ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&             temp_storage_bytes,                        ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        SampleIteratorT     d_samples,                                  ///< [in] The pointer to the input sequence of sample items. The samples from different channels are assumed to be interleaved (e.g., an array of 32-bit pixels where each pixel consists of four RGBA 8-bit samples).
-        CounterT*           d_output_histograms[NUM_ACTIVE_CHANNELS],  ///< [out] The pointers to the histogram counter output arrays, one for each active channel.  For channel<sub><em>i</em></sub>, the allocation length of <tt>d_histograms[i]</tt> should be <tt>num_output_levels[i]</tt> - 1.
-        int                 num_output_levels[NUM_ACTIVE_CHANNELS],     ///< [in] The number of bin level boundaries for delineating histogram samples in each active channel.  Implies that the number of bins for channel<sub><em>i</em></sub> is <tt>num_output_levels[i]</tt> - 1.
-        LevelT              lower_level[NUM_ACTIVE_CHANNELS],           ///< [in] The lower sample value bound (inclusive) for the lowest histogram bin in each active channel.
-        LevelT              upper_level[NUM_ACTIVE_CHANNELS],           ///< [in] The upper sample value bound (exclusive) for the highest histogram bin in each active channel.
-        OffsetT             num_row_pixels,                             ///< [in] The number of multi-channel pixels per row in the region of interest
-        OffsetT             num_rows,                                   ///< [in] The number of rows in the region of interest
-        OffsetT             row_stride_samples,                         ///< [in] The number of samples between starts of consecutive rows in the region of interest
-        cudaStream_t        stream,                                     ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous,                          ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-        Int2Type<true>      is_byte_sample)                             ///< [in] Marker type indicating whether or not SampleT is a 8b type
-    {
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Get PTX version
-            int ptx_version;
-    #if (CUB_PTX_ARCH == 0)
-            if (CubDebug(error = PtxVersion(ptx_version))) break;
-    #else
-            ptx_version = CUB_PTX_ARCH;
-    #endif
-
-            // Get kernel dispatch configurations
-            KernelConfig histogram_sweep_config;
-            if (CubDebug(error = InitConfigs(ptx_version, histogram_sweep_config)))
-                break;
-
-            // Use the pass-thru transform op for converting samples to privatized bins
-            typedef PassThruTransform PrivatizedDecodeOpT;
-
-            // Use the scale transform op for converting privatized bins to output bins
-            typedef ScaleTransform OutputDecodeOpT;
-
-            int                     num_privatized_levels[NUM_ACTIVE_CHANNELS];
-            PrivatizedDecodeOpT     privatized_decode_op[NUM_ACTIVE_CHANNELS];
-            OutputDecodeOpT         output_decode_op[NUM_ACTIVE_CHANNELS];
-            int                     max_levels = num_output_levels[0];
-
-            for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel)
-            {
-                num_privatized_levels[channel] = 257;
-
-                int     bins    = num_output_levels[channel] - 1;
-                LevelT  scale   = (upper_level[channel] - lower_level[channel]) / bins;
-                output_decode_op[channel].Init(num_output_levels[channel], upper_level[channel], lower_level[channel], scale);
-
-                if (num_output_levels[channel] > max_levels)
-                    max_levels = num_output_levels[channel];
-            }
-            int max_num_output_bins = max_levels - 1;
-
-            const int PRIVATIZED_SMEM_BINS = 256;
-
-            if (CubDebug(error = PrivatizedDispatch(
-                d_temp_storage,
-                temp_storage_bytes,
-                d_samples,
-                d_output_histograms,
-                num_privatized_levels,
-                privatized_decode_op,
-                num_output_levels,
-                output_decode_op,
-                max_num_output_bins,
-                num_row_pixels,
-                num_rows,
-                row_stride_samples,
-                DeviceHistogramInitKernel<NUM_ACTIVE_CHANNELS, CounterT, OffsetT>,
-                DeviceHistogramSweepKernel<PtxHistogramSweepPolicy, PRIVATIZED_SMEM_BINS, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, PrivatizedDecodeOpT, OutputDecodeOpT, OffsetT>,
-                histogram_sweep_config,
-                stream,
-                debug_synchronous))) break;
-
-        }
-        while (0);
-
-        return error;
-    }
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/device/dispatch/dispatch_radix_sort.cuh b/pfsimulator/third_party/cub/device/dispatch/dispatch_radix_sort.cuh
deleted file mode 100644
index d1a992d43..000000000
--- a/pfsimulator/third_party/cub/device/dispatch/dispatch_radix_sort.cuh
+++ /dev/null
@@ -1,1619 +0,0 @@
-
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::DeviceRadixSort provides device-wide, parallel operations for computing a radix sort across a sequence of data items residing within device-accessible memory.
- */
-
-#pragma once
-
-#include <stdio.h>
-#include <iterator>
-
-#include "../../agent/agent_radix_sort_upsweep.cuh"
-#include "../../agent/agent_radix_sort_downsweep.cuh"
-#include "../../agent/agent_scan.cuh"
-#include "../../block/block_radix_sort.cuh"
-#include "../../grid/grid_even_share.cuh"
-#include "../../util_type.cuh"
-#include "../../util_debug.cuh"
-#include "../../util_device.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/******************************************************************************
- * Kernel entry points
- *****************************************************************************/
-
-/**
- * Upsweep digit-counting kernel entry point (multi-block).  Computes privatized digit histograms, one per block.
- */
-template <
-    typename                ChainedPolicyT,                 ///< Chained tuning policy
-    bool                    ALT_DIGIT_BITS,                 ///< Whether or not to use the alternate (lower-bits) policy
-    bool                    IS_DESCENDING,                  ///< Whether or not the sorted-order is high-to-low
-    typename                KeyT,                           ///< Key type
-    typename                OffsetT>                        ///< Signed integer type for global offsets
-__launch_bounds__ (int((ALT_DIGIT_BITS) ?
-    ChainedPolicyT::ActivePolicy::AltUpsweepPolicy::BLOCK_THREADS :
-    ChainedPolicyT::ActivePolicy::UpsweepPolicy::BLOCK_THREADS))
-__global__ void DeviceRadixSortUpsweepKernel(
-    const KeyT              *d_keys,                        ///< [in] Input keys buffer
-    OffsetT                 *d_spine,                       ///< [out] Privatized (per block) digit histograms (striped, i.e., 0s counts from each block, then 1s counts from each block, etc.)
-    OffsetT                 /*num_items*/,                  ///< [in] Total number of input data items
-    int                     current_bit,                    ///< [in] Bit position of current radix digit
-    int                     num_bits,                       ///< [in] Number of bits of current radix digit
-    GridEvenShare<OffsetT>  even_share)                     ///< [in] Even-share descriptor for mapan equal number of tiles onto each thread block
-{
-    enum {
-        TILE_ITEMS = ChainedPolicyT::ActivePolicy::AltUpsweepPolicy::BLOCK_THREADS *
-                        ChainedPolicyT::ActivePolicy::AltUpsweepPolicy::ITEMS_PER_THREAD
-    };
-
-    // Parameterize AgentRadixSortUpsweep type for the current configuration
-    typedef AgentRadixSortUpsweep<
-            typename If<(ALT_DIGIT_BITS),
-                typename ChainedPolicyT::ActivePolicy::AltUpsweepPolicy,
-                typename ChainedPolicyT::ActivePolicy::UpsweepPolicy>::Type,
-            KeyT,
-            OffsetT>
-        AgentRadixSortUpsweepT;
-
-    // Shared memory storage
-    __shared__ typename AgentRadixSortUpsweepT::TempStorage temp_storage;
-
-    // Initialize GRID_MAPPING_RAKE even-share descriptor for this thread block
-    even_share.template BlockInit<TILE_ITEMS, GRID_MAPPING_RAKE>();
-
-    AgentRadixSortUpsweepT upsweep(temp_storage, d_keys, current_bit, num_bits);
-
-    upsweep.ProcessRegion(even_share.block_offset, even_share.block_end);
-
-    CTA_SYNC();
-
-    // Write out digit counts (striped)
-    upsweep.template ExtractCounts<IS_DESCENDING>(d_spine, gridDim.x, blockIdx.x);
-}
-
-
-/**
- * Spine scan kernel entry point (single-block).  Computes an exclusive prefix sum over the privatized digit histograms
- */
-template <
-    typename                ChainedPolicyT,                 ///< Chained tuning policy
-    typename                OffsetT>                        ///< Signed integer type for global offsets
-__launch_bounds__ (int(ChainedPolicyT::ActivePolicy::ScanPolicy::BLOCK_THREADS), 1)
-__global__ void RadixSortScanBinsKernel(
-    OffsetT                 *d_spine,                       ///< [in,out] Privatized (per block) digit histograms (striped, i.e., 0s counts from each block, then 1s counts from each block, etc.)
-    int                     num_counts)                     ///< [in] Total number of bin-counts
-{
-    // Parameterize the AgentScan type for the current configuration
-    typedef AgentScan<
-            typename ChainedPolicyT::ActivePolicy::ScanPolicy,
-            OffsetT*,
-            OffsetT*,
-            cub::Sum,
-            OffsetT,
-            OffsetT>
-        AgentScanT;
-
-    // Shared memory storage
-    __shared__ typename AgentScanT::TempStorage temp_storage;
-
-    // Block scan instance
-    AgentScanT block_scan(temp_storage, d_spine, d_spine, cub::Sum(), OffsetT(0)) ;
-
-    // Process full input tiles
-    int block_offset = 0;
-    BlockScanRunningPrefixOp<OffsetT, Sum> prefix_op(0, Sum());
-    while (block_offset + AgentScanT::TILE_ITEMS <= num_counts)
-    {
-        block_scan.template ConsumeTile<false, false>(block_offset, prefix_op);
-        block_offset += AgentScanT::TILE_ITEMS;
-    }
-}
-
-
-/**
- * Downsweep pass kernel entry point (multi-block).  Scatters keys (and values) into corresponding bins for the current digit place.
- */
-template <
-    typename                ChainedPolicyT,                 ///< Chained tuning policy
-    bool                    ALT_DIGIT_BITS,                 ///< Whether or not to use the alternate (lower-bits) policy
-    bool                    IS_DESCENDING,                  ///< Whether or not the sorted-order is high-to-low
-    typename                KeyT,                           ///< Key type
-    typename                ValueT,                         ///< Value type
-    typename                OffsetT>                        ///< Signed integer type for global offsets
-__launch_bounds__ (int((ALT_DIGIT_BITS) ?
-    ChainedPolicyT::ActivePolicy::AltDownsweepPolicy::BLOCK_THREADS :
-    ChainedPolicyT::ActivePolicy::DownsweepPolicy::BLOCK_THREADS))
-__global__ void DeviceRadixSortDownsweepKernel(
-    const KeyT              *d_keys_in,                     ///< [in] Input keys buffer
-    KeyT                    *d_keys_out,                    ///< [in] Output keys buffer
-    const ValueT            *d_values_in,                   ///< [in] Input values buffer
-    ValueT                  *d_values_out,                  ///< [in] Output values buffer
-    OffsetT                 *d_spine,                       ///< [in] Scan of privatized (per block) digit histograms (striped, i.e., 0s counts from each block, then 1s counts from each block, etc.)
-    OffsetT                 num_items,                      ///< [in] Total number of input data items
-    int                     current_bit,                    ///< [in] Bit position of current radix digit
-    int                     num_bits,                       ///< [in] Number of bits of current radix digit
-    GridEvenShare<OffsetT>  even_share)                     ///< [in] Even-share descriptor for mapan equal number of tiles onto each thread block
-{
-    enum {
-        TILE_ITEMS = ChainedPolicyT::ActivePolicy::AltUpsweepPolicy::BLOCK_THREADS *
-                        ChainedPolicyT::ActivePolicy::AltUpsweepPolicy::ITEMS_PER_THREAD
-    };
-
-    // Parameterize AgentRadixSortDownsweep type for the current configuration
-    typedef AgentRadixSortDownsweep<
-            typename If<(ALT_DIGIT_BITS),
-                typename ChainedPolicyT::ActivePolicy::AltDownsweepPolicy,
-                typename ChainedPolicyT::ActivePolicy::DownsweepPolicy>::Type,
-            IS_DESCENDING,
-            KeyT,
-            ValueT,
-            OffsetT>
-        AgentRadixSortDownsweepT;
-
-    // Shared memory storage
-    __shared__  typename AgentRadixSortDownsweepT::TempStorage temp_storage;
-
-    // Initialize even-share descriptor for this thread block
-    even_share.template BlockInit<TILE_ITEMS, GRID_MAPPING_RAKE>();
-
-    // Process input tiles
-    AgentRadixSortDownsweepT(temp_storage, num_items, d_spine, d_keys_in, d_keys_out, d_values_in, d_values_out, current_bit, num_bits).ProcessRegion(
-        even_share.block_offset,
-        even_share.block_end);
-}
-
-
-/**
- * Single pass kernel entry point (single-block).  Fully sorts a tile of input.
- */
-template <
-    typename                ChainedPolicyT,                 ///< Chained tuning policy
-    bool                    IS_DESCENDING,                  ///< Whether or not the sorted-order is high-to-low
-    typename                KeyT,                           ///< Key type
-    typename                ValueT,                         ///< Value type
-    typename                OffsetT>                        ///< Signed integer type for global offsets
-__launch_bounds__ (int(ChainedPolicyT::ActivePolicy::SingleTilePolicy::BLOCK_THREADS), 1)
-__global__ void DeviceRadixSortSingleTileKernel(
-    const KeyT              *d_keys_in,                     ///< [in] Input keys buffer
-    KeyT                    *d_keys_out,                    ///< [in] Output keys buffer
-    const ValueT            *d_values_in,                   ///< [in] Input values buffer
-    ValueT                  *d_values_out,                  ///< [in] Output values buffer
-    OffsetT                 num_items,                      ///< [in] Total number of input data items
-    int                     current_bit,                    ///< [in] Bit position of current radix digit
-    int                     end_bit)                        ///< [in] The past-the-end (most-significant) bit index needed for key comparison
-{
-    // Constants
-    enum
-    {
-        BLOCK_THREADS           = ChainedPolicyT::ActivePolicy::SingleTilePolicy::BLOCK_THREADS,
-        ITEMS_PER_THREAD        = ChainedPolicyT::ActivePolicy::SingleTilePolicy::ITEMS_PER_THREAD,
-        KEYS_ONLY               = Equals<ValueT, NullType>::VALUE,
-    };
-
-    // BlockRadixSort type
-    typedef BlockRadixSort<
-            KeyT,
-            BLOCK_THREADS,
-            ITEMS_PER_THREAD,
-            ValueT,
-            ChainedPolicyT::ActivePolicy::SingleTilePolicy::RADIX_BITS,
-            (ChainedPolicyT::ActivePolicy::SingleTilePolicy::RANK_ALGORITHM == RADIX_RANK_MEMOIZE),
-            ChainedPolicyT::ActivePolicy::SingleTilePolicy::SCAN_ALGORITHM>
-        BlockRadixSortT;
-
-    // BlockLoad type (keys)
-    typedef BlockLoad<
-        KeyT,
-        BLOCK_THREADS,
-        ITEMS_PER_THREAD,
-        ChainedPolicyT::ActivePolicy::SingleTilePolicy::LOAD_ALGORITHM> BlockLoadKeys;
-
-    // BlockLoad type (values)
-    typedef BlockLoad<
-        ValueT,
-        BLOCK_THREADS,
-        ITEMS_PER_THREAD,
-        ChainedPolicyT::ActivePolicy::SingleTilePolicy::LOAD_ALGORITHM> BlockLoadValues;
-
-    // Unsigned word for key bits
-    typedef typename Traits<KeyT>::UnsignedBits UnsignedBitsT;
-
-    // Shared memory storage
-    __shared__ union TempStorage
-    {
-        typename BlockRadixSortT::TempStorage       sort;
-        typename BlockLoadKeys::TempStorage         load_keys;
-        typename BlockLoadValues::TempStorage       load_values;
-
-    } temp_storage;
-
-    // Keys and values for the block
-    KeyT            keys[ITEMS_PER_THREAD];
-    ValueT          values[ITEMS_PER_THREAD];
-
-    // Get default (min/max) value for out-of-bounds keys
-    UnsignedBitsT   default_key_bits = (IS_DESCENDING) ? Traits<KeyT>::LOWEST_KEY : Traits<KeyT>::MAX_KEY;
-    KeyT            default_key = reinterpret_cast<KeyT&>(default_key_bits);
-
-    // Load keys
-    BlockLoadKeys(temp_storage.load_keys).Load(d_keys_in, keys, num_items, default_key);
-
-    CTA_SYNC();
-
-    // Load values
-    if (!KEYS_ONLY)
-    {
-        // Register pressure work-around: moving num_items through shfl prevents compiler
-        // from reusing guards/addressing from prior guarded loads
-        num_items = ShuffleIndex<CUB_PTX_WARP_THREADS>(num_items, 0, 0xffffffff);
-
-        BlockLoadValues(temp_storage.load_values).Load(d_values_in, values, num_items);
-
-        CTA_SYNC();
-    }
-
-    // Sort tile
-    BlockRadixSortT(temp_storage.sort).SortBlockedToStriped(
-        keys,
-        values,
-        current_bit,
-        end_bit,
-        Int2Type<IS_DESCENDING>(),
-        Int2Type<KEYS_ONLY>());
-
-    // Store keys and values
-    #pragma unroll
-    for (int ITEM = 0; ITEM < ITEMS_PER_THREAD; ++ITEM)
-    {
-        int item_offset = ITEM * BLOCK_THREADS + threadIdx.x;
-        if (item_offset < num_items)
-        {
-            d_keys_out[item_offset] = keys[ITEM];
-            if (!KEYS_ONLY)
-                d_values_out[item_offset] = values[ITEM];
-        }
-    }
-}
-
-
-/**
- * Segmented radix sorting pass (one block per segment)
- */
-template <
-    typename                ChainedPolicyT,                 ///< Chained tuning policy
-    bool                    ALT_DIGIT_BITS,                 ///< Whether or not to use the alternate (lower-bits) policy
-    bool                    IS_DESCENDING,                  ///< Whether or not the sorted-order is high-to-low
-    typename                KeyT,                           ///< Key type
-    typename                ValueT,                         ///< Value type
-    typename                OffsetIteratorT,                ///< Random-access input iterator type for reading segment offsets \iterator
-    typename                OffsetT>                        ///< Signed integer type for global offsets
-__launch_bounds__ (int((ALT_DIGIT_BITS) ?
-    ChainedPolicyT::ActivePolicy::AltSegmentedPolicy::BLOCK_THREADS :
-    ChainedPolicyT::ActivePolicy::SegmentedPolicy::BLOCK_THREADS))
-__global__ void DeviceSegmentedRadixSortKernel(
-    const KeyT              *d_keys_in,                     ///< [in] Input keys buffer
-    KeyT                    *d_keys_out,                    ///< [in] Output keys buffer
-    const ValueT            *d_values_in,                   ///< [in] Input values buffer
-    ValueT                  *d_values_out,                  ///< [in] Output values buffer
-    OffsetIteratorT         d_begin_offsets,                ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-    OffsetIteratorT         d_end_offsets,                  ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-    int                     /*num_segments*/,               ///< [in] The number of segments that comprise the sorting data
-    int                     current_bit,                    ///< [in] Bit position of current radix digit
-    int                     pass_bits)                      ///< [in] Number of bits of current radix digit
-{
-    //
-    // Constants
-    //
-
-    typedef typename If<(ALT_DIGIT_BITS),
-        typename ChainedPolicyT::ActivePolicy::AltSegmentedPolicy,
-        typename ChainedPolicyT::ActivePolicy::SegmentedPolicy>::Type SegmentedPolicyT;
-
-    enum
-    {
-        BLOCK_THREADS       = SegmentedPolicyT::BLOCK_THREADS,
-        ITEMS_PER_THREAD    = SegmentedPolicyT::ITEMS_PER_THREAD,
-        RADIX_BITS          = SegmentedPolicyT::RADIX_BITS,
-        TILE_ITEMS          = BLOCK_THREADS * ITEMS_PER_THREAD,
-        RADIX_DIGITS        = 1 << RADIX_BITS,
-        KEYS_ONLY           = Equals<ValueT, NullType>::VALUE,
-    };
-
-    // Upsweep type
-    typedef AgentRadixSortUpsweep<
-            AgentRadixSortUpsweepPolicy<BLOCK_THREADS, ITEMS_PER_THREAD, SegmentedPolicyT::LOAD_MODIFIER, RADIX_BITS>,
-            KeyT,
-            OffsetT>
-        BlockUpsweepT;
-
-    // Digit-scan type
-    typedef BlockScan<OffsetT, BLOCK_THREADS> DigitScanT;
-
-    // Downsweep type
-    typedef AgentRadixSortDownsweep<SegmentedPolicyT, IS_DESCENDING, KeyT, ValueT, OffsetT> BlockDownsweepT;
-
-    enum
-    {
-        /// Number of bin-starting offsets tracked per thread
-        BINS_TRACKED_PER_THREAD = BlockDownsweepT::BINS_TRACKED_PER_THREAD
-    };
-
-    //
-    // Process input tiles
-    //
-
-    // Shared memory storage
-    __shared__ union
-    {
-        typename BlockUpsweepT::TempStorage     upsweep;
-        typename BlockDownsweepT::TempStorage   downsweep;
-        struct
-        {
-            volatile OffsetT                        reverse_counts_in[RADIX_DIGITS];
-            volatile OffsetT                        reverse_counts_out[RADIX_DIGITS];
-            typename DigitScanT::TempStorage        scan;
-        };
-
-    } temp_storage;
-
-    OffsetT segment_begin   = d_begin_offsets[blockIdx.x];
-    OffsetT segment_end     = d_end_offsets[blockIdx.x];
-    OffsetT num_items       = segment_end - segment_begin;
-
-    // Check if empty segment
-    if (num_items <= 0)
-        return;
-
-    // Upsweep
-    BlockUpsweepT upsweep(temp_storage.upsweep, d_keys_in, current_bit, pass_bits);
-    upsweep.ProcessRegion(segment_begin, segment_end);
-
-    CTA_SYNC();
-
-    // The count of each digit value in this pass (valid in the first RADIX_DIGITS threads)
-    OffsetT bin_count[BINS_TRACKED_PER_THREAD];
-    upsweep.ExtractCounts(bin_count);
-
-    CTA_SYNC();
-
-    if (IS_DESCENDING)
-    {
-        // Reverse bin counts
-        #pragma unroll
-        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
-        {
-            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
-
-            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
-                temp_storage.reverse_counts_in[bin_idx] = bin_count[track];
-        }
-
-        CTA_SYNC();
-
-        #pragma unroll
-        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
-        {
-            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
-
-            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
-                bin_count[track] = temp_storage.reverse_counts_in[RADIX_DIGITS - bin_idx - 1];
-        }
-    }
-
-    // Scan
-    OffsetT bin_offset[BINS_TRACKED_PER_THREAD];     // The global scatter base offset for each digit value in this pass (valid in the first RADIX_DIGITS threads)
-    DigitScanT(temp_storage.scan).ExclusiveSum(bin_count, bin_offset);
-
-    #pragma unroll
-    for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
-    {
-        bin_offset[track] += segment_begin;
-    }
-
-    if (IS_DESCENDING)
-    {
-        // Reverse bin offsets
-        #pragma unroll
-        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
-        {
-            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
-
-            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
-                temp_storage.reverse_counts_out[threadIdx.x] = bin_offset[track];
-        }
-
-        CTA_SYNC();
-
-        #pragma unroll
-        for (int track = 0; track < BINS_TRACKED_PER_THREAD; ++track)
-        {
-            int bin_idx = (threadIdx.x * BINS_TRACKED_PER_THREAD) + track;
-
-            if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
-                bin_offset[track] = temp_storage.reverse_counts_out[RADIX_DIGITS - bin_idx - 1];
-        }
-    }
-
-    CTA_SYNC();
-
-    // Downsweep
-    BlockDownsweepT downsweep(temp_storage.downsweep, bin_offset, num_items, d_keys_in, d_keys_out, d_values_in, d_values_out, current_bit, pass_bits);
-    downsweep.ProcessRegion(segment_begin, segment_end);
-}
-
-
-
-/******************************************************************************
- * Policy
- ******************************************************************************/
-
-/**
- * Tuning policy for kernel specialization
- */
-template <
-    typename KeyT,          ///< Key type
-    typename ValueT,        ///< Value type
-    typename OffsetT>       ///< Signed integer type for global offsets
-struct DeviceRadixSortPolicy
-{
-    //------------------------------------------------------------------------------
-    // Constants
-    //------------------------------------------------------------------------------
-
-    enum
-    {
-        // Whether this is a keys-only (or key-value) sort
-        KEYS_ONLY = (Equals<ValueT, NullType>::VALUE),
-    };
-
-    // Dominant-sized key/value type
-    typedef typename If<(sizeof(ValueT) > 4) && (sizeof(KeyT) < sizeof(ValueT)), ValueT, KeyT>::Type DominantT;
-
-    //------------------------------------------------------------------------------
-    // Architecture-specific tuning policies
-    //------------------------------------------------------------------------------
-
-    /// SM20
-    struct Policy200 : ChainedPolicy<200, Policy200, Policy200>
-    {
-        enum {
-            PRIMARY_RADIX_BITS      = 5,
-            ALT_RADIX_BITS          = PRIMARY_RADIX_BITS - 1,
-
-            // Relative size of KeyT type to a 4-byte word
-            SCALE_FACTOR_4B = (CUB_MAX(sizeof(KeyT), sizeof(ValueT)) + 3) / 4,
-        };
-
-        // Keys-only upsweep policies
-        typedef AgentRadixSortUpsweepPolicy <64, CUB_MAX(1, 18 / SCALE_FACTOR_4B), LOAD_DEFAULT, PRIMARY_RADIX_BITS>    UpsweepPolicyKeys;
-        typedef AgentRadixSortUpsweepPolicy <64, CUB_MAX(1, 18 / SCALE_FACTOR_4B), LOAD_DEFAULT, ALT_RADIX_BITS>        AltUpsweepPolicyKeys;
-
-        // Key-value pairs upsweep policies
-        typedef AgentRadixSortUpsweepPolicy <128, CUB_MAX(1, 13 / SCALE_FACTOR_4B), LOAD_DEFAULT, PRIMARY_RADIX_BITS>   UpsweepPolicyPairs;
-        typedef AgentRadixSortUpsweepPolicy <128, CUB_MAX(1, 13 / SCALE_FACTOR_4B), LOAD_DEFAULT, ALT_RADIX_BITS>       AltUpsweepPolicyPairs;
-
-        // Upsweep policies
-        typedef typename If<KEYS_ONLY, UpsweepPolicyKeys, UpsweepPolicyPairs>::Type         UpsweepPolicy;
-        typedef typename If<KEYS_ONLY, AltUpsweepPolicyKeys, AltUpsweepPolicyPairs>::Type   AltUpsweepPolicy;
-
-        // Scan policy
-        typedef AgentScanPolicy <512, 4, BLOCK_LOAD_VECTORIZE, LOAD_DEFAULT, BLOCK_STORE_VECTORIZE, BLOCK_SCAN_RAKING_MEMOIZE> ScanPolicy;
-
-        // Keys-only downsweep policies
-        typedef AgentRadixSortDownsweepPolicy <64, CUB_MAX(1, 18 / SCALE_FACTOR_4B), BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_BASIC, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS>    DownsweepPolicyKeys;
-        typedef AgentRadixSortDownsweepPolicy <64, CUB_MAX(1, 18 / SCALE_FACTOR_4B), BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_BASIC, BLOCK_SCAN_WARP_SCANS, ALT_RADIX_BITS>        AltDownsweepPolicyKeys;
-
-        // Key-value pairs downsweep policies
-        typedef AgentRadixSortDownsweepPolicy <128, CUB_MAX(1, 13 / SCALE_FACTOR_4B), BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_BASIC, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS>   DownsweepPolicyPairs;
-        typedef AgentRadixSortDownsweepPolicy <128, CUB_MAX(1, 13 / SCALE_FACTOR_4B), BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_BASIC, BLOCK_SCAN_WARP_SCANS, ALT_RADIX_BITS>       AltDownsweepPolicyPairs;
-
-        // Downsweep policies
-        typedef typename If<KEYS_ONLY, DownsweepPolicyKeys, DownsweepPolicyPairs>::Type         DownsweepPolicy;
-        typedef typename If<KEYS_ONLY, AltDownsweepPolicyKeys, AltDownsweepPolicyPairs>::Type   AltDownsweepPolicy;
-
-        // Single-tile policy
-        typedef DownsweepPolicy SingleTilePolicy;
-
-        // Segmented policies
-        typedef DownsweepPolicy     SegmentedPolicy;
-        typedef AltDownsweepPolicy  AltSegmentedPolicy;
-    };
-
-    /// SM30
-    struct Policy300 : ChainedPolicy<300, Policy300, Policy200>
-    {
-        enum {
-            PRIMARY_RADIX_BITS      = 5,
-            ALT_RADIX_BITS          = PRIMARY_RADIX_BITS - 1,
-
-            // Relative size of KeyT type to a 4-byte word
-            SCALE_FACTOR_4B = (CUB_MAX(sizeof(KeyT), sizeof(ValueT)) + 3) / 4,
-        };
-
-        // Keys-only upsweep policies
-        typedef AgentRadixSortUpsweepPolicy <256, CUB_MAX(1, 7 / SCALE_FACTOR_4B), LOAD_DEFAULT, PRIMARY_RADIX_BITS>    UpsweepPolicyKeys;
-        typedef AgentRadixSortUpsweepPolicy <256, CUB_MAX(1, 7 / SCALE_FACTOR_4B), LOAD_DEFAULT, ALT_RADIX_BITS>        AltUpsweepPolicyKeys;
-
-        // Key-value pairs upsweep policies
-        typedef AgentRadixSortUpsweepPolicy <256, CUB_MAX(1, 5 / SCALE_FACTOR_4B), LOAD_DEFAULT, PRIMARY_RADIX_BITS>    UpsweepPolicyPairs;
-        typedef AgentRadixSortUpsweepPolicy <256, CUB_MAX(1, 5 / SCALE_FACTOR_4B), LOAD_DEFAULT, ALT_RADIX_BITS>        AltUpsweepPolicyPairs;
-
-        // Upsweep policies
-        typedef typename If<KEYS_ONLY, UpsweepPolicyKeys, UpsweepPolicyPairs>::Type         UpsweepPolicy;
-        typedef typename If<KEYS_ONLY, AltUpsweepPolicyKeys, AltUpsweepPolicyPairs>::Type   AltUpsweepPolicy;
-
-        // Scan policy
-        typedef AgentScanPolicy <1024, 4, BLOCK_LOAD_VECTORIZE, LOAD_DEFAULT, BLOCK_STORE_VECTORIZE, BLOCK_SCAN_WARP_SCANS> ScanPolicy;
-
-        // Keys-only downsweep policies
-        typedef AgentRadixSortDownsweepPolicy <128, CUB_MAX(1, 14 / SCALE_FACTOR_4B), BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_BASIC, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS>   DownsweepPolicyKeys;
-        typedef AgentRadixSortDownsweepPolicy <128, CUB_MAX(1, 14 / SCALE_FACTOR_4B), BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_BASIC, BLOCK_SCAN_WARP_SCANS, ALT_RADIX_BITS>       AltDownsweepPolicyKeys;
-
-        // Key-value pairs downsweep policies
-        typedef AgentRadixSortDownsweepPolicy <128, CUB_MAX(1, 10 / SCALE_FACTOR_4B), BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_BASIC, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS>    DownsweepPolicyPairs;
-        typedef AgentRadixSortDownsweepPolicy <128, CUB_MAX(1, 10 / SCALE_FACTOR_4B), BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_BASIC, BLOCK_SCAN_WARP_SCANS, ALT_RADIX_BITS>        AltDownsweepPolicyPairs;
-
-        // Downsweep policies
-        typedef typename If<KEYS_ONLY, DownsweepPolicyKeys, DownsweepPolicyPairs>::Type         DownsweepPolicy;
-        typedef typename If<KEYS_ONLY, AltDownsweepPolicyKeys, AltDownsweepPolicyPairs>::Type   AltDownsweepPolicy;
-
-        // Single-tile policy
-        typedef DownsweepPolicy SingleTilePolicy;
-
-        // Segmented policies
-        typedef DownsweepPolicy     SegmentedPolicy;
-        typedef AltDownsweepPolicy  AltSegmentedPolicy;
-    };
-
-
-    /// SM35
-    struct Policy350 : ChainedPolicy<350, Policy350, Policy300>
-    {
-        enum {
-            PRIMARY_RADIX_BITS      = (sizeof(KeyT) > 1) ? 6 : 5,    // 1.72B 32b keys/s, 1.17B 32b pairs/s, 1.55B 32b segmented keys/s (K40m)
-        };
-
-        // Scan policy
-        typedef AgentScanPolicy <1024, 4, BLOCK_LOAD_VECTORIZE, LOAD_DEFAULT, BLOCK_STORE_VECTORIZE, BLOCK_SCAN_WARP_SCANS> ScanPolicy;
-
-        // Keys-only downsweep policies
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(128, 9, DominantT), BLOCK_LOAD_WARP_TRANSPOSE, LOAD_LDG, RADIX_RANK_MATCH, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS> DownsweepPolicyKeys;
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(64, 18, DominantT), BLOCK_LOAD_DIRECT, LOAD_LDG, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS - 1> AltDownsweepPolicyKeys;
-
-        // Key-value pairs downsweep policies
-        typedef DownsweepPolicyKeys DownsweepPolicyPairs;
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(128, 15, DominantT), BLOCK_LOAD_DIRECT, LOAD_LDG, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS - 1> AltDownsweepPolicyPairs;
-
-        // Downsweep policies
-        typedef typename If<KEYS_ONLY, DownsweepPolicyKeys, DownsweepPolicyPairs>::Type DownsweepPolicy;
-        typedef typename If<KEYS_ONLY, AltDownsweepPolicyKeys, AltDownsweepPolicyPairs>::Type AltDownsweepPolicy;
-
-        // Upsweep policies
-        typedef DownsweepPolicy UpsweepPolicy;
-        typedef AltDownsweepPolicy AltUpsweepPolicy;
-
-        // Single-tile policy
-        typedef DownsweepPolicy SingleTilePolicy;
-
-        // Segmented policies
-        typedef DownsweepPolicy     SegmentedPolicy;
-        typedef AltDownsweepPolicy  AltSegmentedPolicy;
-
-
-    };
-
-
-    /// SM50
-    struct Policy500 : ChainedPolicy<500, Policy500, Policy350>
-    {
-        enum {
-            PRIMARY_RADIX_BITS      = (sizeof(KeyT) > 1) ? 7 : 5,    // 3.5B 32b keys/s, 1.92B 32b pairs/s (TitanX)
-            SINGLE_TILE_RADIX_BITS  = (sizeof(KeyT) > 1) ? 6 : 5,
-            SEGMENTED_RADIX_BITS    = (sizeof(KeyT) > 1) ? 6 : 5,    // 3.1B 32b segmented keys/s (TitanX)
-        };
-
-        // ScanPolicy
-        typedef AgentScanPolicy <512, 23, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_STORE_WARP_TRANSPOSE, BLOCK_SCAN_RAKING_MEMOIZE> ScanPolicy;
-
-        // Downsweep policies
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(160, 39, DominantT),  BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_BASIC, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS>  DownsweepPolicy;
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 16, DominantT),  BLOCK_LOAD_DIRECT, LOAD_LDG, RADIX_RANK_MEMOIZE, BLOCK_SCAN_RAKING_MEMOIZE, PRIMARY_RADIX_BITS - 1>   AltDownsweepPolicy;
-
-        // Upsweep policies
-        typedef DownsweepPolicy UpsweepPolicy;
-        typedef AltDownsweepPolicy AltUpsweepPolicy;
-
-        // Single-tile policy
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 19, DominantT),  BLOCK_LOAD_DIRECT, LOAD_LDG, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SINGLE_TILE_RADIX_BITS> SingleTilePolicy;
-
-        // Segmented policies
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(192, 31, DominantT),  BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SEGMENTED_RADIX_BITS>   SegmentedPolicy;
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 11, DominantT),  BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SEGMENTED_RADIX_BITS - 1>       AltSegmentedPolicy;
-    };
-
-
-    /// SM60 (GP100)
-    struct Policy600 : ChainedPolicy<600, Policy600, Policy500>
-    {
-        enum {
-            PRIMARY_RADIX_BITS      = (sizeof(KeyT) > 1) ? 7 : 5,    // 6.9B 32b keys/s (Quadro P100)
-            SINGLE_TILE_RADIX_BITS  = (sizeof(KeyT) > 1) ? 6 : 5,
-            SEGMENTED_RADIX_BITS    = (sizeof(KeyT) > 1) ? 6 : 5,    // 5.9B 32b segmented keys/s (Quadro P100)
-        };
-
-        // ScanPolicy
-        typedef AgentScanPolicy <512, 23, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_STORE_WARP_TRANSPOSE, BLOCK_SCAN_RAKING_MEMOIZE> ScanPolicy;
-
-        // Downsweep policies
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 25, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MATCH, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS>   DownsweepPolicy;
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(192, 39, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS - 1>   AltDownsweepPolicy;
-
-        // Upsweep policies
-        typedef DownsweepPolicy UpsweepPolicy;
-        typedef AltDownsweepPolicy AltUpsweepPolicy;
-
-        // Single-tile policy
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 19, DominantT),  BLOCK_LOAD_DIRECT, LOAD_LDG, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SINGLE_TILE_RADIX_BITS>          SingleTilePolicy;
-
-        // Segmented policies
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(192, 39, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SEGMENTED_RADIX_BITS>     SegmentedPolicy;
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(384, 11, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SEGMENTED_RADIX_BITS - 1> AltSegmentedPolicy;
-
-    };
-
-
-    /// SM61 (GP104)
-    struct Policy610 : ChainedPolicy<610, Policy610, Policy600>
-    {
-        enum {
-            PRIMARY_RADIX_BITS      = (sizeof(KeyT) > 1) ? 7 : 5,    // 3.4B 32b keys/s, 1.83B 32b pairs/s (1080)
-            SINGLE_TILE_RADIX_BITS  = (sizeof(KeyT) > 1) ? 6 : 5,
-            SEGMENTED_RADIX_BITS    = (sizeof(KeyT) > 1) ? 6 : 5,    // 3.3B 32b segmented keys/s (1080)
-        };
-
-        // ScanPolicy
-        typedef AgentScanPolicy <512, 23, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_STORE_WARP_TRANSPOSE, BLOCK_SCAN_RAKING_MEMOIZE> ScanPolicy;
-
-        // Downsweep policies
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(384, 31, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MATCH, BLOCK_SCAN_RAKING_MEMOIZE, PRIMARY_RADIX_BITS>   DownsweepPolicy;
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 35, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_RAKING_MEMOIZE, PRIMARY_RADIX_BITS - 1>   AltDownsweepPolicy;
-
-        // Upsweep policies
-        typedef AgentRadixSortUpsweepPolicy <CUB_SCALED_GRANULARITIES(128, 16, DominantT), LOAD_LDG, PRIMARY_RADIX_BITS>        UpsweepPolicy;
-        typedef AgentRadixSortUpsweepPolicy <CUB_SCALED_GRANULARITIES(128, 16, DominantT), LOAD_LDG, PRIMARY_RADIX_BITS - 1>    AltUpsweepPolicy;
-
-        // Single-tile policy
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 19, DominantT),  BLOCK_LOAD_DIRECT, LOAD_LDG, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SINGLE_TILE_RADIX_BITS>          SingleTilePolicy;
-
-        // Segmented policies
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(192, 39, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SEGMENTED_RADIX_BITS>     SegmentedPolicy;
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(384, 11, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SEGMENTED_RADIX_BITS - 1> AltSegmentedPolicy;
-    };
-
-
-    /// SM62 (Tegra, less RF)
-    struct Policy620 : ChainedPolicy<620, Policy620, Policy610>
-    {
-        enum {
-            PRIMARY_RADIX_BITS      = 5,
-            ALT_RADIX_BITS          = PRIMARY_RADIX_BITS - 1,
-        };
-
-        // ScanPolicy
-        typedef AgentScanPolicy <512, 23, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_STORE_WARP_TRANSPOSE, BLOCK_SCAN_RAKING_MEMOIZE> ScanPolicy;
-
-        // Downsweep policies
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 16, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_RAKING_MEMOIZE, PRIMARY_RADIX_BITS>   DownsweepPolicy;
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 16, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_RAKING_MEMOIZE, ALT_RADIX_BITS>       AltDownsweepPolicy;
-
-        // Upsweep policies
-        typedef DownsweepPolicy UpsweepPolicy;
-        typedef AltDownsweepPolicy AltUpsweepPolicy;
-
-        // Single-tile policy
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 19, DominantT),  BLOCK_LOAD_DIRECT, LOAD_LDG, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS> SingleTilePolicy;
-
-        // Segmented policies
-        typedef DownsweepPolicy     SegmentedPolicy;
-        typedef AltDownsweepPolicy  AltSegmentedPolicy;
-    };
-
-
-    /// SM70 (GV100)
-    struct Policy700 : ChainedPolicy<700, Policy700, Policy620>
-    {
-        enum {
-            PRIMARY_RADIX_BITS      = (sizeof(KeyT) > 1) ? 7 : 5,    // 7.62B 32b keys/s (GV100)
-            SINGLE_TILE_RADIX_BITS  = (sizeof(KeyT) > 1) ? 6 : 5,
-            SEGMENTED_RADIX_BITS    = (sizeof(KeyT) > 1) ? 6 : 5,    // 8.7B 32b segmented keys/s (GV100)
-        };
-
-        // ScanPolicy
-        typedef AgentScanPolicy <512, 23, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_STORE_WARP_TRANSPOSE, BLOCK_SCAN_RAKING_MEMOIZE> ScanPolicy;
-
-        // Downsweep policies
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 25, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MATCH, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS>   DownsweepPolicy;
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 25, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, PRIMARY_RADIX_BITS - 1>   AltDownsweepPolicy;
-
-        // Upsweep policies
-        typedef DownsweepPolicy UpsweepPolicy;
-        typedef AltDownsweepPolicy AltUpsweepPolicy;
-
-        // Single-tile policy
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(256, 19, DominantT),  BLOCK_LOAD_DIRECT, LOAD_LDG, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SINGLE_TILE_RADIX_BITS>          SingleTilePolicy;
-
-        // Segmented policies
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(192, 39, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SEGMENTED_RADIX_BITS>     SegmentedPolicy;
-        typedef AgentRadixSortDownsweepPolicy <CUB_SCALED_GRANULARITIES(384, 11, DominantT),  BLOCK_LOAD_TRANSPOSE, LOAD_DEFAULT, RADIX_RANK_MEMOIZE, BLOCK_SCAN_WARP_SCANS, SEGMENTED_RADIX_BITS - 1> AltSegmentedPolicy;
-    };
-
-
-    /// MaxPolicy
-    typedef Policy700 MaxPolicy;
-
-
-};
-
-
-
-/******************************************************************************
- * Single-problem dispatch
- ******************************************************************************/
-
-/**
- * Utility class for dispatching the appropriately-tuned kernels for device-wide radix sort
- */
-template <
-    bool     IS_DESCENDING, ///< Whether or not the sorted-order is high-to-low
-    typename KeyT,          ///< Key type
-    typename ValueT,        ///< Value type
-    typename OffsetT>       ///< Signed integer type for global offsets
-struct DispatchRadixSort :
-    DeviceRadixSortPolicy<KeyT, ValueT, OffsetT>
-{
-    //------------------------------------------------------------------------------
-    // Constants
-    //------------------------------------------------------------------------------
-
-    enum
-    {
-        // Whether this is a keys-only (or key-value) sort
-        KEYS_ONLY = (Equals<ValueT, NullType>::VALUE),
-    };
-
-
-    //------------------------------------------------------------------------------
-    // Problem state
-    //------------------------------------------------------------------------------
-
-    void                    *d_temp_storage;        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-    size_t                  &temp_storage_bytes;    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-    DoubleBuffer<KeyT>      &d_keys;                ///< [in,out] Double-buffer whose current buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
-    DoubleBuffer<ValueT>    &d_values;              ///< [in,out] Double-buffer whose current buffer contains the unsorted input values and, upon return, is updated to point to the sorted output values
-    OffsetT                 num_items;              ///< [in] Number of items to sort
-    int                     begin_bit;              ///< [in] The beginning (least-significant) bit index needed for key comparison
-    int                     end_bit;                ///< [in] The past-the-end (most-significant) bit index needed for key comparison
-    cudaStream_t            stream;                 ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-    bool                    debug_synchronous;      ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    int                     ptx_version;            ///< [in] PTX version
-    bool                    is_overwrite_okay;      ///< [in] Whether is okay to overwrite source buffers
-
-
-    //------------------------------------------------------------------------------
-    // Constructor
-    //------------------------------------------------------------------------------
-
-    /// Constructor
-    CUB_RUNTIME_FUNCTION __forceinline__
-    DispatchRadixSort(
-        void*                   d_temp_storage,
-        size_t                  &temp_storage_bytes,
-        DoubleBuffer<KeyT>      &d_keys,
-        DoubleBuffer<ValueT>    &d_values,
-        OffsetT                 num_items,
-        int                     begin_bit,
-        int                     end_bit,
-        bool                    is_overwrite_okay,
-        cudaStream_t            stream,
-        bool                    debug_synchronous,
-        int                     ptx_version)
-    :
-        d_temp_storage(d_temp_storage),
-        temp_storage_bytes(temp_storage_bytes),
-        d_keys(d_keys),
-        d_values(d_values),
-        num_items(num_items),
-        begin_bit(begin_bit),
-        end_bit(end_bit),
-        stream(stream),
-        debug_synchronous(debug_synchronous),
-        ptx_version(ptx_version),
-        is_overwrite_okay(is_overwrite_okay)
-    {}
-
-
-    //------------------------------------------------------------------------------
-    // Small-problem (single tile) invocation
-    //------------------------------------------------------------------------------
-
-    /// Invoke a single block to sort in-core
-    template <
-        typename                ActivePolicyT,          ///< Umbrella policy active for the target device
-        typename                SingleTileKernelT>      ///< Function type of cub::DeviceRadixSortSingleTileKernel
-    CUB_RUNTIME_FUNCTION __forceinline__
-    cudaError_t InvokeSingleTile(
-        SingleTileKernelT       single_tile_kernel)     ///< [in] Kernel function pointer to parameterization of cub::DeviceRadixSortSingleTileKernel
-    {
-#ifndef CUB_RUNTIME_ENABLED
-        (void)single_tile_kernel;
-        // Kernel launch not supported from this device
-        return CubDebug(cudaErrorNotSupported );
-#else
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Return if the caller is simply requesting the size of the storage allocation
-            if (d_temp_storage == NULL)
-            {
-                temp_storage_bytes = 1;
-                break;
-            }
-
-            // Return if empty problem
-            if (num_items == 0)
-                break;
-
-            // Log single_tile_kernel configuration
-            if (debug_synchronous)
-                _CubLog("Invoking single_tile_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy, current bit %d, bit_grain %d\n",
-                    1, ActivePolicyT::SingleTilePolicy::BLOCK_THREADS, (long long) stream,
-                    ActivePolicyT::SingleTilePolicy::ITEMS_PER_THREAD, 1, begin_bit, ActivePolicyT::SingleTilePolicy::RADIX_BITS);
-
-            // Invoke upsweep_kernel with same grid size as downsweep_kernel
-            single_tile_kernel<<<1, ActivePolicyT::SingleTilePolicy::BLOCK_THREADS, 0, stream>>>(
-                d_keys.Current(),
-                d_keys.Alternate(),
-                d_values.Current(),
-                d_values.Alternate(),
-                num_items,
-                begin_bit,
-                end_bit);
-
-            // Check for failure to launch
-            if (CubDebug(error = cudaPeekAtLastError())) break;
-
-            // Sync the stream if specified to flush runtime errors
-            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-
-            // Update selector
-            d_keys.selector ^= 1;
-            d_values.selector ^= 1;
-        }
-        while (0);
-
-        return error;
-
-#endif // CUB_RUNTIME_ENABLED
-    }
-
-
-    //------------------------------------------------------------------------------
-    // Normal problem size invocation
-    //------------------------------------------------------------------------------
-
-    /**
-     * Invoke a three-kernel sorting pass at the current bit.
-     */
-    template <typename PassConfigT>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    cudaError_t InvokePass(
-        const KeyT      *d_keys_in,
-        KeyT            *d_keys_out,
-        const ValueT    *d_values_in,
-        ValueT          *d_values_out,
-        OffsetT         *d_spine,
-        int             spine_length,
-        int             &current_bit,
-        PassConfigT     &pass_config)
-    {
-        cudaError error = cudaSuccess;
-        do
-        {
-            int pass_bits = CUB_MIN(pass_config.radix_bits, (end_bit - current_bit));
-
-            // Log upsweep_kernel configuration
-            if (debug_synchronous)
-                _CubLog("Invoking upsweep_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy, current bit %d, bit_grain %d\n",
-                pass_config.even_share.grid_size, pass_config.upsweep_config.block_threads, (long long) stream,
-                pass_config.upsweep_config.items_per_thread, pass_config.upsweep_config.sm_occupancy, current_bit, pass_bits);
-
-            // Invoke upsweep_kernel with same grid size as downsweep_kernel
-            pass_config.upsweep_kernel<<<pass_config.even_share.grid_size, pass_config.upsweep_config.block_threads, 0, stream>>>(
-                d_keys_in,
-                d_spine,
-                num_items,
-                current_bit,
-                pass_bits,
-                pass_config.even_share);
-
-            // Check for failure to launch
-            if (CubDebug(error = cudaPeekAtLastError())) break;
-
-            // Sync the stream if specified to flush runtime errors
-            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-
-            // Log scan_kernel configuration
-            if (debug_synchronous) _CubLog("Invoking scan_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread\n",
-                1, pass_config.scan_config.block_threads, (long long) stream, pass_config.scan_config.items_per_thread);
-
-            // Invoke scan_kernel
-            pass_config.scan_kernel<<<1, pass_config.scan_config.block_threads, 0, stream>>>(
-                d_spine,
-                spine_length);
-
-            // Check for failure to launch
-            if (CubDebug(error = cudaPeekAtLastError())) break;
-
-            // Sync the stream if specified to flush runtime errors
-            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-
-            // Log downsweep_kernel configuration
-            if (debug_synchronous) _CubLog("Invoking downsweep_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
-                pass_config.even_share.grid_size, pass_config.downsweep_config.block_threads, (long long) stream,
-                pass_config.downsweep_config.items_per_thread, pass_config.downsweep_config.sm_occupancy);
-
-            // Invoke downsweep_kernel
-            pass_config.downsweep_kernel<<<pass_config.even_share.grid_size, pass_config.downsweep_config.block_threads, 0, stream>>>(
-                d_keys_in,
-                d_keys_out,
-                d_values_in,
-                d_values_out,
-                d_spine,
-                num_items,
-                current_bit,
-                pass_bits,
-                pass_config.even_share);
-
-            // Check for failure to launch
-            if (CubDebug(error = cudaPeekAtLastError())) break;
-
-            // Sync the stream if specified to flush runtime errors
-            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-
-            // Update current bit
-            current_bit += pass_bits;
-        }
-        while (0);
-
-        return error;
-    }
-
-
-
-    /// Pass configuration structure
-    template <
-        typename UpsweepKernelT,
-        typename ScanKernelT,
-        typename DownsweepKernelT>
-    struct PassConfig
-    {
-        UpsweepKernelT          upsweep_kernel;
-        KernelConfig            upsweep_config;
-        ScanKernelT             scan_kernel;
-        KernelConfig            scan_config;
-        DownsweepKernelT        downsweep_kernel;
-        KernelConfig            downsweep_config;
-        int                     radix_bits;
-        int                     radix_digits;
-        int                     max_downsweep_grid_size;
-        GridEvenShare<OffsetT>  even_share;
-
-        /// Initialize pass configuration
-        template <
-            typename UpsweepPolicyT,
-            typename ScanPolicyT,
-            typename DownsweepPolicyT>
-        CUB_RUNTIME_FUNCTION __forceinline__
-        cudaError_t InitPassConfig(
-            UpsweepKernelT      upsweep_kernel,
-            ScanKernelT         scan_kernel,
-            DownsweepKernelT    downsweep_kernel,
-            int                 ptx_version,
-            int                 sm_count,
-            int                 num_items)
-        {
-            cudaError error = cudaSuccess;
-            do
-            {
-                this->upsweep_kernel    = upsweep_kernel;
-                this->scan_kernel       = scan_kernel;
-                this->downsweep_kernel  = downsweep_kernel;
-                radix_bits              = DownsweepPolicyT::RADIX_BITS;
-                radix_digits            = 1 << radix_bits;
-
-                if (CubDebug(error = upsweep_config.Init<UpsweepPolicyT>(upsweep_kernel))) break;
-                if (CubDebug(error = scan_config.Init<ScanPolicyT>(scan_kernel))) break;
-                if (CubDebug(error = downsweep_config.Init<DownsweepPolicyT>(downsweep_kernel))) break;
-
-                max_downsweep_grid_size = (downsweep_config.sm_occupancy * sm_count) * CUB_SUBSCRIPTION_FACTOR(ptx_version);
-
-                even_share.DispatchInit(
-                    num_items,
-                    max_downsweep_grid_size,
-                    CUB_MAX(downsweep_config.tile_size, upsweep_config.tile_size));
-
-            }
-            while (0);
-            return error;
-        }
-
-    };
-
-
-    /// Invocation (run multiple digit passes)
-    template <
-        typename            ActivePolicyT,          ///< Umbrella policy active for the target device
-        typename            UpsweepKernelT,         ///< Function type of cub::DeviceRadixSortUpsweepKernel
-        typename            ScanKernelT,            ///< Function type of cub::SpineScanKernel
-        typename            DownsweepKernelT>       ///< Function type of cub::DeviceRadixSortDownsweepKernel
-    CUB_RUNTIME_FUNCTION __forceinline__
-    cudaError_t InvokePasses(
-        UpsweepKernelT      upsweep_kernel,         ///< [in] Kernel function pointer to parameterization of cub::DeviceRadixSortUpsweepKernel
-        UpsweepKernelT      alt_upsweep_kernel,     ///< [in] Alternate kernel function pointer to parameterization of cub::DeviceRadixSortUpsweepKernel
-        ScanKernelT         scan_kernel,            ///< [in] Kernel function pointer to parameterization of cub::SpineScanKernel
-        DownsweepKernelT    downsweep_kernel,       ///< [in] Kernel function pointer to parameterization of cub::DeviceRadixSortDownsweepKernel
-        DownsweepKernelT    alt_downsweep_kernel)   ///< [in] Alternate kernel function pointer to parameterization of cub::DeviceRadixSortDownsweepKernel
-    {
-#ifndef CUB_RUNTIME_ENABLED
-        (void)upsweep_kernel;
-        (void)alt_upsweep_kernel;
-        (void)scan_kernel;
-        (void)downsweep_kernel;
-        (void)alt_downsweep_kernel;
-
-        // Kernel launch not supported from this device
-        return CubDebug(cudaErrorNotSupported );
-#else
-
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Get device ordinal
-            int device_ordinal;
-            if (CubDebug(error = cudaGetDevice(&device_ordinal))) break;
-
-            // Get SM count
-            int sm_count;
-            if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break;
-
-            // Init regular and alternate-digit kernel configurations
-            PassConfig<UpsweepKernelT, ScanKernelT, DownsweepKernelT> pass_config, alt_pass_config;
-            if ((error = pass_config.template InitPassConfig<
-                    typename ActivePolicyT::UpsweepPolicy, 
-                    typename ActivePolicyT::ScanPolicy, 
-                    typename ActivePolicyT::DownsweepPolicy>(
-                upsweep_kernel, scan_kernel, downsweep_kernel, ptx_version, sm_count, num_items))) break;
-
-            if ((error = alt_pass_config.template InitPassConfig<
-                    typename ActivePolicyT::AltUpsweepPolicy, 
-                    typename ActivePolicyT::ScanPolicy, 
-                    typename ActivePolicyT::AltDownsweepPolicy>(
-                alt_upsweep_kernel, scan_kernel, alt_downsweep_kernel, ptx_version, sm_count, num_items))) break;
-
-            // Get maximum spine length
-            int max_grid_size       = CUB_MAX(pass_config.max_downsweep_grid_size, alt_pass_config.max_downsweep_grid_size);
-            int spine_length        = (max_grid_size * pass_config.radix_digits) + pass_config.scan_config.tile_size;
-
-            // Temporary storage allocation requirements
-            void* allocations[3];
-            size_t allocation_sizes[3] =
-            {
-                spine_length * sizeof(OffsetT),                                         // bytes needed for privatized block digit histograms
-                (is_overwrite_okay) ? 0 : num_items * sizeof(KeyT),                     // bytes needed for 3rd keys buffer
-                (is_overwrite_okay || (KEYS_ONLY)) ? 0 : num_items * sizeof(ValueT),    // bytes needed for 3rd values buffer
-            };
-
-            // Alias the temporary allocations from the single storage blob (or compute the necessary size of the blob)
-            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
-
-            // Return if the caller is simply requesting the size of the storage allocation
-            if (d_temp_storage == NULL)
-                return cudaSuccess;
-
-            // Pass planning.  Run passes of the alternate digit-size configuration until we have an even multiple of our preferred digit size
-            int num_bits            = end_bit - begin_bit;
-            int num_passes          = (num_bits + pass_config.radix_bits - 1) / pass_config.radix_bits;
-            bool is_num_passes_odd  = num_passes & 1;
-            int max_alt_passes      = (num_passes * pass_config.radix_bits) - num_bits;
-            int alt_end_bit         = CUB_MIN(end_bit, begin_bit + (max_alt_passes * alt_pass_config.radix_bits));
-
-            // Alias the temporary storage allocations
-            OffsetT *d_spine = static_cast<OffsetT*>(allocations[0]);
-
-            DoubleBuffer<KeyT> d_keys_remaining_passes(
-                (is_overwrite_okay || is_num_passes_odd) ? d_keys.Alternate() : static_cast<KeyT*>(allocations[1]),
-                (is_overwrite_okay) ? d_keys.Current() : (is_num_passes_odd) ? static_cast<KeyT*>(allocations[1]) : d_keys.Alternate());
-
-            DoubleBuffer<ValueT> d_values_remaining_passes(
-                (is_overwrite_okay || is_num_passes_odd) ? d_values.Alternate() : static_cast<ValueT*>(allocations[2]),
-                (is_overwrite_okay) ? d_values.Current() : (is_num_passes_odd) ? static_cast<ValueT*>(allocations[2]) : d_values.Alternate());
-
-            // Run first pass, consuming from the input's current buffers
-            int current_bit = begin_bit;
-            if (CubDebug(error = InvokePass(
-                d_keys.Current(), d_keys_remaining_passes.Current(),
-                d_values.Current(), d_values_remaining_passes.Current(),
-                d_spine, spine_length, current_bit,
-                (current_bit < alt_end_bit) ? alt_pass_config : pass_config))) break;
-
-            // Run remaining passes
-            while (current_bit < end_bit)
-            {
-                if (CubDebug(error = InvokePass(
-                    d_keys_remaining_passes.d_buffers[d_keys_remaining_passes.selector],    d_keys_remaining_passes.d_buffers[d_keys_remaining_passes.selector ^ 1],
-                    d_values_remaining_passes.d_buffers[d_keys_remaining_passes.selector],  d_values_remaining_passes.d_buffers[d_keys_remaining_passes.selector ^ 1],
-                    d_spine, spine_length, current_bit,
-                    (current_bit < alt_end_bit) ? alt_pass_config : pass_config))) break;;
-
-                // Invert selectors
-                d_keys_remaining_passes.selector ^= 1;
-                d_values_remaining_passes.selector ^= 1;
-            }
-
-            // Update selector
-            if (!is_overwrite_okay) {
-                num_passes = 1; // Sorted data always ends up in the other vector
-            }
-
-            d_keys.selector = (d_keys.selector + num_passes) & 1;
-            d_values.selector = (d_values.selector + num_passes) & 1;
-        }
-        while (0);
-
-        return error;
-
-#endif // CUB_RUNTIME_ENABLED
-    }
-
-
-    //------------------------------------------------------------------------------
-    // Chained policy invocation
-    //------------------------------------------------------------------------------
-
-    /// Invocation
-    template <typename ActivePolicyT>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    cudaError_t Invoke()
-    {
-        typedef typename DispatchRadixSort::MaxPolicy       MaxPolicyT;
-        typedef typename ActivePolicyT::SingleTilePolicy    SingleTilePolicyT;
-
-        // Force kernel code-generation in all compiler passes
-        if (num_items <= (SingleTilePolicyT::BLOCK_THREADS * SingleTilePolicyT::ITEMS_PER_THREAD))
-        {
-            // Small, single tile size
-            return InvokeSingleTile<ActivePolicyT>(
-                DeviceRadixSortSingleTileKernel<MaxPolicyT, IS_DESCENDING, KeyT, ValueT, OffsetT>);
-        }
-        else
-        {
-            // Regular size
-            return InvokePasses<ActivePolicyT>(
-                DeviceRadixSortUpsweepKernel<   MaxPolicyT, false,   IS_DESCENDING, KeyT, OffsetT>,
-                DeviceRadixSortUpsweepKernel<   MaxPolicyT, true,    IS_DESCENDING, KeyT, OffsetT>,
-                RadixSortScanBinsKernel<        MaxPolicyT, OffsetT>,
-                DeviceRadixSortDownsweepKernel< MaxPolicyT, false,   IS_DESCENDING, KeyT, ValueT, OffsetT>,
-                DeviceRadixSortDownsweepKernel< MaxPolicyT, true,    IS_DESCENDING, KeyT, ValueT, OffsetT>);
-        }
-    }
-
-
-    //------------------------------------------------------------------------------
-    // Dispatch entrypoints
-    //------------------------------------------------------------------------------
-
-    /**
-     * Internal dispatch routine
-     */
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Dispatch(
-        void*                   d_temp_storage,         ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                  &temp_storage_bytes,    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        DoubleBuffer<KeyT>      &d_keys,                ///< [in,out] Double-buffer whose current buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
-        DoubleBuffer<ValueT>    &d_values,              ///< [in,out] Double-buffer whose current buffer contains the unsorted input values and, upon return, is updated to point to the sorted output values
-        OffsetT                 num_items,              ///< [in] Number of items to sort
-        int                     begin_bit,              ///< [in] The beginning (least-significant) bit index needed for key comparison
-        int                     end_bit,                ///< [in] The past-the-end (most-significant) bit index needed for key comparison
-        bool                    is_overwrite_okay,      ///< [in] Whether is okay to overwrite source buffers
-        cudaStream_t            stream,                 ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                    debug_synchronous)      ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        typedef typename DispatchRadixSort::MaxPolicy MaxPolicyT;
-
-        cudaError_t error;
-        do {
-            // Get PTX version
-            int ptx_version;
-            if (CubDebug(error = PtxVersion(ptx_version))) break;
-
-            // Create dispatch functor
-            DispatchRadixSort dispatch(
-                d_temp_storage, temp_storage_bytes,
-                d_keys, d_values,
-                num_items, begin_bit, end_bit, is_overwrite_okay,
-                stream, debug_synchronous, ptx_version);
-
-            // Dispatch to chained policy
-            if (CubDebug(error = MaxPolicyT::Invoke(ptx_version, dispatch))) break;
-
-        } while (0);
-
-        return error;
-    }
-};
-
-
-
-
-/******************************************************************************
- * Segmented dispatch
- ******************************************************************************/
-
-/**
- * Utility class for dispatching the appropriately-tuned kernels for segmented device-wide radix sort
- */
-template <
-    bool     IS_DESCENDING,     ///< Whether or not the sorted-order is high-to-low
-    typename KeyT,              ///< Key type
-    typename ValueT,            ///< Value type
-    typename OffsetIteratorT,   ///< Random-access input iterator type for reading segment offsets \iterator
-    typename OffsetT>           ///< Signed integer type for global offsets
-struct DispatchSegmentedRadixSort :
-    DeviceRadixSortPolicy<KeyT, ValueT, OffsetT>
-{
-    //------------------------------------------------------------------------------
-    // Constants
-    //------------------------------------------------------------------------------
-
-    enum
-    {
-        // Whether this is a keys-only (or key-value) sort
-        KEYS_ONLY = (Equals<ValueT, NullType>::VALUE),
-    };
-
-
-    //------------------------------------------------------------------------------
-    // Parameter members
-    //------------------------------------------------------------------------------
-
-    void                    *d_temp_storage;        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-    size_t                  &temp_storage_bytes;    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-    DoubleBuffer<KeyT>      &d_keys;                ///< [in,out] Double-buffer whose current buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
-    DoubleBuffer<ValueT>    &d_values;              ///< [in,out] Double-buffer whose current buffer contains the unsorted input values and, upon return, is updated to point to the sorted output values
-    OffsetT                 num_items;              ///< [in] Number of items to sort
-    OffsetT                 num_segments;           ///< [in] The number of segments that comprise the sorting data
-    OffsetIteratorT         d_begin_offsets;        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-    OffsetIteratorT         d_end_offsets;          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-    int                     begin_bit;              ///< [in] The beginning (least-significant) bit index needed for key comparison
-    int                     end_bit;                ///< [in] The past-the-end (most-significant) bit index needed for key comparison
-    cudaStream_t            stream;                 ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-    bool                    debug_synchronous;      ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    int                     ptx_version;            ///< [in] PTX version
-    bool                    is_overwrite_okay;      ///< [in] Whether is okay to overwrite source buffers
-
-
-    //------------------------------------------------------------------------------
-    // Constructors
-    //------------------------------------------------------------------------------
-
-    /// Constructor
-    CUB_RUNTIME_FUNCTION __forceinline__
-    DispatchSegmentedRadixSort(
-        void*                   d_temp_storage,
-        size_t                  &temp_storage_bytes,
-        DoubleBuffer<KeyT>      &d_keys,
-        DoubleBuffer<ValueT>    &d_values,
-        OffsetT                 num_items,
-        OffsetT                 num_segments,
-        OffsetIteratorT         d_begin_offsets,
-        OffsetIteratorT         d_end_offsets,
-        int                     begin_bit,
-        int                     end_bit,
-        bool                    is_overwrite_okay,
-        cudaStream_t            stream,
-        bool                    debug_synchronous,
-        int                     ptx_version)
-    :
-        d_temp_storage(d_temp_storage),
-        temp_storage_bytes(temp_storage_bytes),
-        d_keys(d_keys),
-        d_values(d_values),
-        num_items(num_items),
-        num_segments(num_segments),
-        d_begin_offsets(d_begin_offsets),
-        d_end_offsets(d_end_offsets),
-        begin_bit(begin_bit),
-        end_bit(end_bit),
-        is_overwrite_okay(is_overwrite_okay),
-        stream(stream),
-        debug_synchronous(debug_synchronous),
-        ptx_version(ptx_version)
-    {}
-
-
-    //------------------------------------------------------------------------------
-    // Multi-segment invocation
-    //------------------------------------------------------------------------------
-
-    /// Invoke a three-kernel sorting pass at the current bit.
-    template <typename PassConfigT>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    cudaError_t InvokePass(
-        const KeyT      *d_keys_in,
-        KeyT            *d_keys_out,
-        const ValueT    *d_values_in,
-        ValueT          *d_values_out,
-        int             &current_bit,
-        PassConfigT     &pass_config)
-    {
-        cudaError error = cudaSuccess;
-        do
-        {
-            int pass_bits = CUB_MIN(pass_config.radix_bits, (end_bit - current_bit));
-
-            // Log kernel configuration
-            if (debug_synchronous)
-                _CubLog("Invoking segmented_kernels<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy, current bit %d, bit_grain %d\n",
-                    num_segments, pass_config.segmented_config.block_threads, (long long) stream,
-                pass_config.segmented_config.items_per_thread, pass_config.segmented_config.sm_occupancy, current_bit, pass_bits);
-
-            pass_config.segmented_kernel<<<num_segments, pass_config.segmented_config.block_threads, 0, stream>>>(
-                d_keys_in, d_keys_out,
-                d_values_in,  d_values_out,
-                d_begin_offsets, d_end_offsets, num_segments,
-                current_bit, pass_bits);
-
-            // Check for failure to launch
-            if (CubDebug(error = cudaPeekAtLastError())) break;
-
-            // Sync the stream if specified to flush runtime errors
-            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-
-            // Update current bit
-            current_bit += pass_bits;
-        }
-        while (0);
-
-        return error;
-    }
-
-
-    /// PassConfig data structure
-    template <typename SegmentedKernelT>
-    struct PassConfig
-    {
-        SegmentedKernelT    segmented_kernel;
-        KernelConfig        segmented_config;
-        int                 radix_bits;
-        int                 radix_digits;
-
-        /// Initialize pass configuration
-        template <typename SegmentedPolicyT>
-        CUB_RUNTIME_FUNCTION __forceinline__
-        cudaError_t InitPassConfig(SegmentedKernelT segmented_kernel)
-        {
-            this->segmented_kernel  = segmented_kernel;
-            this->radix_bits        = SegmentedPolicyT::RADIX_BITS;
-            this->radix_digits      = 1 << radix_bits;
-
-            return CubDebug(segmented_config.Init<SegmentedPolicyT>(segmented_kernel));
-        }
-    };
-
-
-    /// Invocation (run multiple digit passes)
-    template <
-        typename                ActivePolicyT,          ///< Umbrella policy active for the target device
-        typename                SegmentedKernelT>       ///< Function type of cub::DeviceSegmentedRadixSortKernel
-    CUB_RUNTIME_FUNCTION __forceinline__
-    cudaError_t InvokePasses(
-        SegmentedKernelT     segmented_kernel,          ///< [in] Kernel function pointer to parameterization of cub::DeviceSegmentedRadixSortKernel
-        SegmentedKernelT     alt_segmented_kernel)      ///< [in] Alternate kernel function pointer to parameterization of cub::DeviceSegmentedRadixSortKernel
-    {
-#ifndef CUB_RUNTIME_ENABLED
-      (void)segmented_kernel;
-      (void)alt_segmented_kernel;
-
-        // Kernel launch not supported from this device
-        return CubDebug(cudaErrorNotSupported );
-#else
-
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Init regular and alternate kernel configurations
-            PassConfig<SegmentedKernelT> pass_config, alt_pass_config;
-            if ((error = pass_config.template       InitPassConfig<typename ActivePolicyT::SegmentedPolicy>(segmented_kernel))) break;
-            if ((error = alt_pass_config.template   InitPassConfig<typename ActivePolicyT::AltSegmentedPolicy>(alt_segmented_kernel))) break;
-
-            // Temporary storage allocation requirements
-            void* allocations[2];
-            size_t allocation_sizes[2] =
-            {
-                (is_overwrite_okay) ? 0 : num_items * sizeof(KeyT),                      // bytes needed for 3rd keys buffer
-                (is_overwrite_okay || (KEYS_ONLY)) ? 0 : num_items * sizeof(ValueT),     // bytes needed for 3rd values buffer
-            };
-
-            // Alias the temporary allocations from the single storage blob (or compute the necessary size of the blob)
-            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
-
-            // Return if the caller is simply requesting the size of the storage allocation
-            if (d_temp_storage == NULL)
-            {
-                if (temp_storage_bytes == 0)
-                    temp_storage_bytes = 1;
-                return cudaSuccess;
-            }
-
-            // Pass planning.  Run passes of the alternate digit-size configuration until we have an even multiple of our preferred digit size
-            int radix_bits          = ActivePolicyT::SegmentedPolicy::RADIX_BITS;
-            int alt_radix_bits      = ActivePolicyT::AltSegmentedPolicy::RADIX_BITS;
-            int num_bits            = end_bit - begin_bit;
-            int num_passes          = (num_bits + radix_bits - 1) / radix_bits;
-            bool is_num_passes_odd  = num_passes & 1;
-            int max_alt_passes      = (num_passes * radix_bits) - num_bits;
-            int alt_end_bit         = CUB_MIN(end_bit, begin_bit + (max_alt_passes * alt_radix_bits));
-
-            DoubleBuffer<KeyT> d_keys_remaining_passes(
-                (is_overwrite_okay || is_num_passes_odd) ? d_keys.Alternate() : static_cast<KeyT*>(allocations[0]),
-                (is_overwrite_okay) ? d_keys.Current() : (is_num_passes_odd) ? static_cast<KeyT*>(allocations[0]) : d_keys.Alternate());
-
-            DoubleBuffer<ValueT> d_values_remaining_passes(
-                (is_overwrite_okay || is_num_passes_odd) ? d_values.Alternate() : static_cast<ValueT*>(allocations[1]),
-                (is_overwrite_okay) ? d_values.Current() : (is_num_passes_odd) ? static_cast<ValueT*>(allocations[1]) : d_values.Alternate());
-
-            // Run first pass, consuming from the input's current buffers
-            int current_bit = begin_bit;
-
-            if (CubDebug(error = InvokePass(
-                d_keys.Current(), d_keys_remaining_passes.Current(),
-                d_values.Current(), d_values_remaining_passes.Current(),
-                current_bit,
-                (current_bit < alt_end_bit) ? alt_pass_config : pass_config))) break;
-
-            // Run remaining passes
-            while (current_bit < end_bit)
-            {
-                if (CubDebug(error = InvokePass(
-                    d_keys_remaining_passes.d_buffers[d_keys_remaining_passes.selector],    d_keys_remaining_passes.d_buffers[d_keys_remaining_passes.selector ^ 1],
-                    d_values_remaining_passes.d_buffers[d_keys_remaining_passes.selector],  d_values_remaining_passes.d_buffers[d_keys_remaining_passes.selector ^ 1],
-                    current_bit,
-                    (current_bit < alt_end_bit) ? alt_pass_config : pass_config))) break;
-
-                // Invert selectors and update current bit
-                d_keys_remaining_passes.selector ^= 1;
-                d_values_remaining_passes.selector ^= 1;
-            }
-
-            // Update selector
-            if (!is_overwrite_okay) {
-                num_passes = 1; // Sorted data always ends up in the other vector
-            }
-
-            d_keys.selector = (d_keys.selector + num_passes) & 1;
-            d_values.selector = (d_values.selector + num_passes) & 1;
-        }
-        while (0);
-
-        return error;
-
-#endif // CUB_RUNTIME_ENABLED
-    }
-
-
-    //------------------------------------------------------------------------------
-    // Chained policy invocation
-    //------------------------------------------------------------------------------
-
-    /// Invocation
-    template <typename ActivePolicyT>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    cudaError_t Invoke()
-    {
-        typedef typename DispatchSegmentedRadixSort::MaxPolicy MaxPolicyT;
-
-        // Force kernel code-generation in all compiler passes
-        return InvokePasses<ActivePolicyT>(
-            DeviceSegmentedRadixSortKernel<MaxPolicyT, false,   IS_DESCENDING, KeyT, ValueT, OffsetIteratorT, OffsetT>,
-            DeviceSegmentedRadixSortKernel<MaxPolicyT, true,    IS_DESCENDING, KeyT, ValueT, OffsetIteratorT, OffsetT>);
-    }
-
-
-    //------------------------------------------------------------------------------
-    // Dispatch entrypoints
-    //------------------------------------------------------------------------------
-
-
-    /// Internal dispatch routine
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Dispatch(
-        void*                   d_temp_storage,         ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t                  &temp_storage_bytes,    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        DoubleBuffer<KeyT>      &d_keys,                ///< [in,out] Double-buffer whose current buffer contains the unsorted input keys and, upon return, is updated to point to the sorted output keys
-        DoubleBuffer<ValueT>    &d_values,              ///< [in,out] Double-buffer whose current buffer contains the unsorted input values and, upon return, is updated to point to the sorted output values
-        int                     num_items,              ///< [in] Number of items to sort
-        int                     num_segments,           ///< [in] The number of segments that comprise the sorting data
-        OffsetIteratorT         d_begin_offsets,        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-        OffsetIteratorT         d_end_offsets,          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-        int                     begin_bit,              ///< [in] The beginning (least-significant) bit index needed for key comparison
-        int                     end_bit,                ///< [in] The past-the-end (most-significant) bit index needed for key comparison
-        bool                    is_overwrite_okay,      ///< [in] Whether is okay to overwrite source buffers
-        cudaStream_t            stream,                 ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                    debug_synchronous)      ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        typedef typename DispatchSegmentedRadixSort::MaxPolicy MaxPolicyT;
-
-        cudaError_t error;
-        do {
-            // Get PTX version
-            int ptx_version;
-            if (CubDebug(error = PtxVersion(ptx_version))) break;
-
-            // Create dispatch functor
-            DispatchSegmentedRadixSort dispatch(
-                d_temp_storage, temp_storage_bytes,
-                d_keys, d_values,
-                num_items, num_segments, d_begin_offsets, d_end_offsets,
-                begin_bit, end_bit, is_overwrite_okay,
-                stream, debug_synchronous, ptx_version);
-
-            // Dispatch to chained policy
-            if (CubDebug(error = MaxPolicyT::Invoke(ptx_version, dispatch))) break;
-
-        } while (0);
-
-        return error;
-    }
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/device/dispatch/dispatch_reduce.cuh b/pfsimulator/third_party/cub/device/dispatch/dispatch_reduce.cuh
deleted file mode 100644
index e9d1b7ac1..000000000
--- a/pfsimulator/third_party/cub/device/dispatch/dispatch_reduce.cuh
+++ /dev/null
@@ -1,882 +0,0 @@
-
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::DeviceReduce provides device-wide, parallel operations for computing a reduction across a sequence of data items residing within device-accessible memory.
- */
-
-#pragma once
-
-#include <stdio.h>
-#include <iterator>
-
-#include "../../agent/agent_reduce.cuh"
-#include "../../iterator/arg_index_input_iterator.cuh"
-#include "../../thread/thread_operators.cuh"
-#include "../../grid/grid_even_share.cuh"
-#include "../../iterator/arg_index_input_iterator.cuh"
-#include "../../util_debug.cuh"
-#include "../../util_device.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/******************************************************************************
- * Kernel entry points
- *****************************************************************************/
-
-/**
- * Reduce region kernel entry point (multi-block).  Computes privatized reductions, one per thread block.
- */
-template <
-    typename                ChainedPolicyT,             ///< Chained tuning policy
-    typename                InputIteratorT,             ///< Random-access input iterator type for reading input items \iterator
-    typename                OutputIteratorT,            ///< Output iterator type for recording the reduced aggregate \iterator
-    typename                OffsetT,                    ///< Signed integer type for global offsets
-    typename                ReductionOpT>               ///< Binary reduction functor type having member <tt>T operator()(const T &a, const T &b)</tt>
-__launch_bounds__ (int(ChainedPolicyT::ActivePolicy::ReducePolicy::BLOCK_THREADS))
-__global__ void DeviceReduceKernel(
-    InputIteratorT          d_in,                       ///< [in] Pointer to the input sequence of data items
-    OutputIteratorT         d_out,                      ///< [out] Pointer to the output aggregate
-    OffsetT                 num_items,                  ///< [in] Total number of input data items
-    GridEvenShare<OffsetT>  even_share,                 ///< [in] Even-share descriptor for mapping an equal number of tiles onto each thread block
-    ReductionOpT            reduction_op)               ///< [in] Binary reduction functor
-{
-    // The output value type
-    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
-        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
-        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
-
-    // Thread block type for reducing input tiles
-    typedef AgentReduce<
-            typename ChainedPolicyT::ActivePolicy::ReducePolicy,
-            InputIteratorT,
-            OutputIteratorT,
-            OffsetT,
-            ReductionOpT>
-        AgentReduceT;
-
-    // Shared memory storage
-    __shared__ typename AgentReduceT::TempStorage temp_storage;
-
-    // Consume input tiles
-    OutputT block_aggregate = AgentReduceT(temp_storage, d_in, reduction_op).ConsumeTiles(even_share);
-
-    // Output result
-    if (threadIdx.x == 0)
-        d_out[blockIdx.x] = block_aggregate;
-}
-
-
-/**
- * Reduce a single tile kernel entry point (single-block).  Can be used to aggregate privatized thread block reductions from a previous multi-block reduction pass.
- */
-template <
-    typename                ChainedPolicyT,             ///< Chained tuning policy
-    typename                InputIteratorT,             ///< Random-access input iterator type for reading input items \iterator
-    typename                OutputIteratorT,            ///< Output iterator type for recording the reduced aggregate \iterator
-    typename                OffsetT,                    ///< Signed integer type for global offsets
-    typename                ReductionOpT,               ///< Binary reduction functor type having member <tt>T operator()(const T &a, const T &b)</tt>
-    typename                OuputT>                     ///< Data element type that is convertible to the \p value type of \p OutputIteratorT
-__launch_bounds__ (int(ChainedPolicyT::ActivePolicy::SingleTilePolicy::BLOCK_THREADS), 1)
-__global__ void DeviceReduceSingleTileKernel(
-    InputIteratorT          d_in,                       ///< [in] Pointer to the input sequence of data items
-    OutputIteratorT         d_out,                      ///< [out] Pointer to the output aggregate
-    OffsetT                 num_items,                  ///< [in] Total number of input data items
-    ReductionOpT            reduction_op,               ///< [in] Binary reduction functor
-    OuputT                  init)                       ///< [in] The initial value of the reduction
-{
-    // Thread block type for reducing input tiles
-    typedef AgentReduce<
-            typename ChainedPolicyT::ActivePolicy::SingleTilePolicy,
-            InputIteratorT,
-            OutputIteratorT,
-            OffsetT,
-            ReductionOpT>
-        AgentReduceT;
-
-    // Shared memory storage
-    __shared__ typename AgentReduceT::TempStorage temp_storage;
-
-    // Check if empty problem
-    if (num_items == 0)
-    {
-        if (threadIdx.x == 0)
-            *d_out = init;
-        return;
-    }
-
-    // Consume input tiles
-    OuputT block_aggregate = AgentReduceT(temp_storage, d_in, reduction_op).ConsumeRange(
-        OffsetT(0),
-        num_items);
-
-    // Output result
-    if (threadIdx.x == 0)
-        *d_out = reduction_op(init, block_aggregate);
-}
-
-
-/// Normalize input iterator to segment offset
-template <typename T, typename OffsetT, typename IteratorT>
-__device__ __forceinline__
-void NormalizeReductionOutput(
-    T &/*val*/,
-    OffsetT /*base_offset*/,
-    IteratorT /*itr*/)
-{}
-
-
-/// Normalize input iterator to segment offset (specialized for arg-index)
-template <typename KeyValuePairT, typename OffsetT, typename WrappedIteratorT, typename OutputValueT>
-__device__ __forceinline__
-void NormalizeReductionOutput(
-    KeyValuePairT &val,
-    OffsetT base_offset,
-    ArgIndexInputIterator<WrappedIteratorT, OffsetT, OutputValueT> /*itr*/)
-{
-    val.key -= base_offset;
-}
-
-
-/**
- * Segmented reduction (one block per segment)
- */
-template <
-    typename                ChainedPolicyT,             ///< Chained tuning policy
-    typename                InputIteratorT,             ///< Random-access input iterator type for reading input items \iterator
-    typename                OutputIteratorT,            ///< Output iterator type for recording the reduced aggregate \iterator
-    typename                OffsetIteratorT,            ///< Random-access input iterator type for reading segment offsets \iterator
-    typename                OffsetT,                    ///< Signed integer type for global offsets
-    typename                ReductionOpT,               ///< Binary reduction functor type having member <tt>T operator()(const T &a, const T &b)</tt>
-    typename                OutputT>                    ///< Data element type that is convertible to the \p value type of \p OutputIteratorT
-__launch_bounds__ (int(ChainedPolicyT::ActivePolicy::ReducePolicy::BLOCK_THREADS))
-__global__ void DeviceSegmentedReduceKernel(
-    InputIteratorT          d_in,                       ///< [in] Pointer to the input sequence of data items
-    OutputIteratorT         d_out,                      ///< [out] Pointer to the output aggregate
-    OffsetIteratorT         d_begin_offsets,            ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-    OffsetIteratorT         d_end_offsets,              ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-    int                     /*num_segments*/,           ///< [in] The number of segments that comprise the sorting data
-    ReductionOpT            reduction_op,               ///< [in] Binary reduction functor 
-    OutputT                 init)                       ///< [in] The initial value of the reduction
-{
-    // Thread block type for reducing input tiles
-    typedef AgentReduce<
-            typename ChainedPolicyT::ActivePolicy::ReducePolicy,
-            InputIteratorT,
-            OutputIteratorT,
-            OffsetT,
-            ReductionOpT>
-        AgentReduceT;
-
-    // Shared memory storage
-    __shared__ typename AgentReduceT::TempStorage temp_storage;
-
-    OffsetT segment_begin   = d_begin_offsets[blockIdx.x];
-    OffsetT segment_end     = d_end_offsets[blockIdx.x];
-
-    // Check if empty problem
-    if (segment_begin == segment_end)
-    {
-        if (threadIdx.x == 0)
-            d_out[blockIdx.x] = init;
-        return;
-    }
-
-    // Consume input tiles
-    OutputT block_aggregate = AgentReduceT(temp_storage, d_in, reduction_op).ConsumeRange(
-        segment_begin,
-        segment_end);
-
-    // Normalize as needed
-    NormalizeReductionOutput(block_aggregate, segment_begin, d_in);
-
-    if (threadIdx.x == 0)
-        d_out[blockIdx.x] = reduction_op(init, block_aggregate);;
-}
-
-
-
-
-/******************************************************************************
- * Policy
- ******************************************************************************/
-
-template <
-    typename OuputT,            ///< Data type
-    typename OffsetT,           ///< Signed integer type for global offsets
-    typename ReductionOpT>      ///< Binary reduction functor type having member <tt>T operator()(const T &a, const T &b)</tt> 
-struct DeviceReducePolicy
-{
-    //------------------------------------------------------------------------------
-    // Architecture-specific tuning policies
-    //------------------------------------------------------------------------------
-
-    /// SM13
-    struct Policy130 : ChainedPolicy<130, Policy130, Policy130>
-    {
-        // ReducePolicy
-        typedef AgentReducePolicy<
-                CUB_SCALED_GRANULARITIES(128, 8, OuputT), ///< Threads per block, items per thread
-                2,                                  ///< Number of items per vectorized load
-                BLOCK_REDUCE_RAKING,                ///< Cooperative block-wide reduction algorithm to use
-                LOAD_DEFAULT>                       ///< Cache load modifier
-            ReducePolicy;
-
-        // SingleTilePolicy
-        typedef ReducePolicy SingleTilePolicy;
-
-        // SegmentedReducePolicy
-        typedef ReducePolicy SegmentedReducePolicy;
-    };
-
-
-    /// SM20
-    struct Policy200 : ChainedPolicy<200, Policy200, Policy130>
-    {
-        // ReducePolicy (GTX 580: 178.9 GB/s @ 48M 4B items, 158.1 GB/s @ 192M 1B items)
-        typedef AgentReducePolicy<
-                CUB_SCALED_GRANULARITIES(128, 8, OuputT),     ///< Threads per block, items per thread
-                4,                                      ///< Number of items per vectorized load
-                BLOCK_REDUCE_RAKING,                    ///< Cooperative block-wide reduction algorithm to use
-                LOAD_DEFAULT>                           ///< Cache load modifier
-            ReducePolicy;
-
-        // SingleTilePolicy
-        typedef ReducePolicy SingleTilePolicy;
-
-        // SegmentedReducePolicy
-        typedef ReducePolicy SegmentedReducePolicy;
-    };
-
-
-    /// SM30
-    struct Policy300 : ChainedPolicy<300, Policy300, Policy200>
-    {
-        // ReducePolicy (GTX670: 154.0 @ 48M 4B items)
-        typedef AgentReducePolicy<
-                CUB_SCALED_GRANULARITIES(256, 20, OuputT),    ///< Threads per block, items per thread
-                2,                                      ///< Number of items per vectorized load
-                BLOCK_REDUCE_WARP_REDUCTIONS,           ///< Cooperative block-wide reduction algorithm to use
-                LOAD_DEFAULT>                           ///< Cache load modifier
-            ReducePolicy;
-
-        // SingleTilePolicy
-        typedef ReducePolicy SingleTilePolicy;
-
-        // SegmentedReducePolicy
-        typedef ReducePolicy SegmentedReducePolicy;
-    };
-
-
-    /// SM35
-    struct Policy350 : ChainedPolicy<350, Policy350, Policy300>
-    {
-        // ReducePolicy (GTX Titan: 255.1 GB/s @ 48M 4B items; 228.7 GB/s @ 192M 1B items)
-        typedef AgentReducePolicy<
-                CUB_SCALED_GRANULARITIES(256, 20, OuputT),    ///< Threads per block, items per thread
-                4,                                      ///< Number of items per vectorized load
-                BLOCK_REDUCE_WARP_REDUCTIONS,           ///< Cooperative block-wide reduction algorithm to use
-                LOAD_LDG>                               ///< Cache load modifier
-            ReducePolicy;
-
-        // SingleTilePolicy
-        typedef ReducePolicy SingleTilePolicy;
-
-        // SegmentedReducePolicy
-        typedef ReducePolicy SegmentedReducePolicy;
-    };
-
-    /// SM60
-    struct Policy600 : ChainedPolicy<600, Policy600, Policy350>
-    {
-        // ReducePolicy (P100: 591 GB/s @ 64M 4B items; 583 GB/s @ 256M 1B items)
-        typedef AgentReducePolicy<
-                CUB_SCALED_GRANULARITIES(256, 16, OuputT),    ///< Threads per block, items per thread
-                4,                                      ///< Number of items per vectorized load
-                BLOCK_REDUCE_WARP_REDUCTIONS,           ///< Cooperative block-wide reduction algorithm to use
-                LOAD_LDG>                               ///< Cache load modifier
-            ReducePolicy;
-
-        // SingleTilePolicy
-        typedef ReducePolicy SingleTilePolicy;
-
-        // SegmentedReducePolicy
-        typedef ReducePolicy SegmentedReducePolicy;
-    };
-
-
-    /// MaxPolicy
-    typedef Policy600 MaxPolicy;
-
-};
-
-
-
-/******************************************************************************
- * Single-problem dispatch
- ******************************************************************************/
-
-/**
- * Utility class for dispatching the appropriately-tuned kernels for device-wide reduction
- */
-template <
-    typename InputIteratorT,    ///< Random-access input iterator type for reading input items \iterator
-    typename OutputIteratorT,   ///< Output iterator type for recording the reduced aggregate \iterator
-    typename OffsetT,           ///< Signed integer type for global offsets
-    typename ReductionOpT>      ///< Binary reduction functor type having member <tt>T operator()(const T &a, const T &b)</tt> 
-struct DispatchReduce :
-    DeviceReducePolicy<
-        typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
-            typename std::iterator_traits<InputIteratorT>::value_type,                                  // ... then the input iterator's value type,
-            typename std::iterator_traits<OutputIteratorT>::value_type>::Type,                          // ... else the output iterator's value type
-        OffsetT,
-        ReductionOpT>
-{
-    //------------------------------------------------------------------------------
-    // Constants
-    //------------------------------------------------------------------------------
-
-    // Data type of output iterator
-    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
-        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
-        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
-
-
-    //------------------------------------------------------------------------------
-    // Problem state
-    //------------------------------------------------------------------------------
-
-    void                *d_temp_storage;                ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-    size_t              &temp_storage_bytes;            ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-    InputIteratorT      d_in;                           ///< [in] Pointer to the input sequence of data items
-    OutputIteratorT     d_out;                          ///< [out] Pointer to the output aggregate
-    OffsetT             num_items;                      ///< [in] Total number of input items (i.e., length of \p d_in)
-    ReductionOpT        reduction_op;                   ///< [in] Binary reduction functor 
-    OutputT             init;                           ///< [in] The initial value of the reduction
-    cudaStream_t        stream;                         ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-    bool                debug_synchronous;              ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    int                 ptx_version;                    ///< [in] PTX version
-
-    //------------------------------------------------------------------------------
-    // Constructor
-    //------------------------------------------------------------------------------
-
-    /// Constructor
-    CUB_RUNTIME_FUNCTION __forceinline__
-    DispatchReduce(
-        void*                   d_temp_storage,
-        size_t                  &temp_storage_bytes,
-        InputIteratorT          d_in,
-        OutputIteratorT         d_out,
-        OffsetT                 num_items,
-        ReductionOpT            reduction_op,
-        OutputT                 init,
-        cudaStream_t            stream,
-        bool                    debug_synchronous,
-        int                     ptx_version)
-    :
-        d_temp_storage(d_temp_storage),
-        temp_storage_bytes(temp_storage_bytes),
-        d_in(d_in),
-        d_out(d_out),
-        num_items(num_items),
-        reduction_op(reduction_op),
-        init(init),
-        stream(stream),
-        debug_synchronous(debug_synchronous),
-        ptx_version(ptx_version)
-    {}
-
-
-    //------------------------------------------------------------------------------
-    // Small-problem (single tile) invocation
-    //------------------------------------------------------------------------------
-
-    /// Invoke a single block block to reduce in-core
-    template <
-        typename                ActivePolicyT,          ///< Umbrella policy active for the target device
-        typename                SingleTileKernelT>      ///< Function type of cub::DeviceReduceSingleTileKernel
-    CUB_RUNTIME_FUNCTION __forceinline__
-    cudaError_t InvokeSingleTile(
-        SingleTileKernelT       single_tile_kernel)     ///< [in] Kernel function pointer to parameterization of cub::DeviceReduceSingleTileKernel
-    {
-#ifndef CUB_RUNTIME_ENABLED
-        (void)single_tile_kernel;
-
-        // Kernel launch not supported from this device
-        return CubDebug(cudaErrorNotSupported );
-#else
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Return if the caller is simply requesting the size of the storage allocation
-            if (d_temp_storage == NULL)
-            {
-                temp_storage_bytes = 1;
-                break;
-            }
-
-            // Log single_reduce_sweep_kernel configuration
-            if (debug_synchronous) _CubLog("Invoking DeviceReduceSingleTileKernel<<<1, %d, 0, %lld>>>(), %d items per thread\n",
-                ActivePolicyT::SingleTilePolicy::BLOCK_THREADS,
-                (long long) stream,
-                ActivePolicyT::SingleTilePolicy::ITEMS_PER_THREAD);
-
-            // Invoke single_reduce_sweep_kernel
-            single_tile_kernel<<<1, ActivePolicyT::SingleTilePolicy::BLOCK_THREADS, 0, stream>>>(
-                d_in,
-                d_out,
-                num_items,
-                reduction_op,
-                init);
-
-            // Check for failure to launch
-            if (CubDebug(error = cudaPeekAtLastError())) break;
-
-            // Sync the stream if specified to flush runtime errors
-            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-        }
-        while (0);
-
-        return error;
-
-#endif // CUB_RUNTIME_ENABLED
-    }
-
-
-    //------------------------------------------------------------------------------
-    // Normal problem size invocation (two-pass)
-    //------------------------------------------------------------------------------
-
-    /// Invoke two-passes to reduce
-    template <
-        typename                ActivePolicyT,              ///< Umbrella policy active for the target device
-        typename                ReduceKernelT,              ///< Function type of cub::DeviceReduceKernel
-        typename                SingleTileKernelT>          ///< Function type of cub::DeviceReduceSingleTileKernel
-    CUB_RUNTIME_FUNCTION __forceinline__
-    cudaError_t InvokePasses(
-        ReduceKernelT           reduce_kernel,          ///< [in] Kernel function pointer to parameterization of cub::DeviceReduceKernel
-        SingleTileKernelT       single_tile_kernel)     ///< [in] Kernel function pointer to parameterization of cub::DeviceReduceSingleTileKernel
-    {
-#ifndef CUB_RUNTIME_ENABLED
-        (void)                  reduce_kernel;
-        (void)                  single_tile_kernel;
-
-        // Kernel launch not supported from this device
-        return CubDebug(cudaErrorNotSupported );
-#else
-
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Get device ordinal
-            int device_ordinal;
-            if (CubDebug(error = cudaGetDevice(&device_ordinal))) break;
-
-            // Get SM count
-            int sm_count;
-            if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break;
-
-            // Init regular kernel configuration
-            KernelConfig reduce_config;
-            if (CubDebug(error = reduce_config.Init<typename ActivePolicyT::ReducePolicy>(reduce_kernel))) break;
-            int reduce_device_occupancy = reduce_config.sm_occupancy * sm_count;
-
-            // Even-share work distribution
-            int max_blocks = reduce_device_occupancy * CUB_SUBSCRIPTION_FACTOR(ptx_version);
-            GridEvenShare<OffsetT> even_share;
-            even_share.DispatchInit(num_items, max_blocks, reduce_config.tile_size);
-
-            // Temporary storage allocation requirements
-            void* allocations[1];
-            size_t allocation_sizes[1] =
-            {
-                max_blocks * sizeof(OutputT)    // bytes needed for privatized block reductions
-            };
-
-            // Alias the temporary allocations from the single storage blob (or compute the necessary size of the blob)
-            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
-            if (d_temp_storage == NULL)
-            {
-                // Return if the caller is simply requesting the size of the storage allocation
-                return cudaSuccess;
-            }
-
-            // Alias the allocation for the privatized per-block reductions
-            OutputT *d_block_reductions = (OutputT*) allocations[0];
-
-            // Get grid size for device_reduce_sweep_kernel
-            int reduce_grid_size = even_share.grid_size;
-
-            // Log device_reduce_sweep_kernel configuration
-            if (debug_synchronous) _CubLog("Invoking DeviceReduceKernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
-                reduce_grid_size,
-                ActivePolicyT::ReducePolicy::BLOCK_THREADS,
-                (long long) stream,
-                ActivePolicyT::ReducePolicy::ITEMS_PER_THREAD,
-                reduce_config.sm_occupancy);
-
-            // Invoke DeviceReduceKernel
-            reduce_kernel<<<reduce_grid_size, ActivePolicyT::ReducePolicy::BLOCK_THREADS, 0, stream>>>(
-                d_in,
-                d_block_reductions,
-                num_items,
-                even_share,
-                reduction_op);
-
-            // Check for failure to launch
-            if (CubDebug(error = cudaPeekAtLastError())) break;
-
-            // Sync the stream if specified to flush runtime errors
-            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-
-            // Log single_reduce_sweep_kernel configuration
-            if (debug_synchronous) _CubLog("Invoking DeviceReduceSingleTileKernel<<<1, %d, 0, %lld>>>(), %d items per thread\n",
-                ActivePolicyT::SingleTilePolicy::BLOCK_THREADS,
-                (long long) stream,
-                ActivePolicyT::SingleTilePolicy::ITEMS_PER_THREAD);
-
-            // Invoke DeviceReduceSingleTileKernel
-            single_tile_kernel<<<1, ActivePolicyT::SingleTilePolicy::BLOCK_THREADS, 0, stream>>>(
-                d_block_reductions,
-                d_out,
-                reduce_grid_size,
-                reduction_op,
-                init);
-
-            // Check for failure to launch
-            if (CubDebug(error = cudaPeekAtLastError())) break;
-
-            // Sync the stream if specified to flush runtime errors
-            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-        }
-        while (0);
-
-        return error;
-
-#endif // CUB_RUNTIME_ENABLED
-
-    }
-
-
-    //------------------------------------------------------------------------------
-    // Chained policy invocation
-    //------------------------------------------------------------------------------
-
-    /// Invocation
-    template <typename ActivePolicyT>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    cudaError_t Invoke()
-    {
-        typedef typename ActivePolicyT::SingleTilePolicy    SingleTilePolicyT;
-        typedef typename DispatchReduce::MaxPolicy          MaxPolicyT;
-
-        // Force kernel code-generation in all compiler passes
-        if (num_items <= (SingleTilePolicyT::BLOCK_THREADS * SingleTilePolicyT::ITEMS_PER_THREAD))
-        {
-            // Small, single tile size
-            return InvokeSingleTile<ActivePolicyT>(
-                DeviceReduceSingleTileKernel<MaxPolicyT, InputIteratorT, OutputIteratorT, OffsetT, ReductionOpT, OutputT>);
-        }
-        else
-        {
-            // Regular size
-            return InvokePasses<ActivePolicyT>(
-                DeviceReduceKernel<typename DispatchReduce::MaxPolicy, InputIteratorT, OutputT*, OffsetT, ReductionOpT>,
-                DeviceReduceSingleTileKernel<MaxPolicyT, OutputT*, OutputIteratorT, OffsetT, ReductionOpT, OutputT>);
-        }
-    }
-
-
-    //------------------------------------------------------------------------------
-    // Dispatch entrypoints
-    //------------------------------------------------------------------------------
-
-    /**
-     * Internal dispatch routine for computing a device-wide reduction
-     */
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Dispatch(
-        void            *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t          &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT  d_in,                               ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT d_out,                              ///< [out] Pointer to the output aggregate
-        OffsetT         num_items,                          ///< [in] Total number of input items (i.e., length of \p d_in)
-        ReductionOpT    reduction_op,                       ///< [in] Binary reduction functor 
-        OutputT         init,                               ///< [in] The initial value of the reduction
-        cudaStream_t    stream,                             ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool            debug_synchronous)                  ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        typedef typename DispatchReduce::MaxPolicy MaxPolicyT;
-
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Get PTX version
-            int ptx_version;
-            if (CubDebug(error = PtxVersion(ptx_version))) break;
-
-            // Create dispatch functor
-            DispatchReduce dispatch(
-                d_temp_storage, temp_storage_bytes,
-                d_in, d_out, num_items, reduction_op, init,
-                stream, debug_synchronous, ptx_version);
-
-            // Dispatch to chained policy
-            if (CubDebug(error = MaxPolicyT::Invoke(ptx_version, dispatch))) break;
-        }
-        while (0);
-
-        return error;
-    }
-};
-
-
-
-/******************************************************************************
- * Segmented dispatch
- ******************************************************************************/
-
-/**
- * Utility class for dispatching the appropriately-tuned kernels for device-wide reduction
- */
-template <
-    typename InputIteratorT,    ///< Random-access input iterator type for reading input items \iterator
-    typename OutputIteratorT,   ///< Output iterator type for recording the reduced aggregate \iterator
-    typename OffsetIteratorT,   ///< Random-access input iterator type for reading segment offsets \iterator
-    typename OffsetT,           ///< Signed integer type for global offsets
-    typename ReductionOpT>      ///< Binary reduction functor type having member <tt>T operator()(const T &a, const T &b)</tt> 
-struct DispatchSegmentedReduce :
-    DeviceReducePolicy<
-        typename std::iterator_traits<InputIteratorT>::value_type,
-        OffsetT,
-        ReductionOpT>
-{
-    //------------------------------------------------------------------------------
-    // Constants
-    //------------------------------------------------------------------------------
-
-    /// The output value type
-    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
-        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
-        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
-
-
-    //------------------------------------------------------------------------------
-    // Problem state
-    //------------------------------------------------------------------------------
-
-    void                *d_temp_storage;        ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-    size_t              &temp_storage_bytes;    ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-    InputIteratorT      d_in;                   ///< [in] Pointer to the input sequence of data items
-    OutputIteratorT     d_out;                  ///< [out] Pointer to the output aggregate
-    OffsetT             num_segments;           ///< [in] The number of segments that comprise the sorting data
-    OffsetIteratorT     d_begin_offsets;        ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-    OffsetIteratorT     d_end_offsets;          ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-    ReductionOpT        reduction_op;           ///< [in] Binary reduction functor 
-    OutputT             init;                   ///< [in] The initial value of the reduction
-    cudaStream_t        stream;                 ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-    bool                debug_synchronous;      ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    int                 ptx_version;            ///< [in] PTX version
-
-    //------------------------------------------------------------------------------
-    // Constructor
-    //------------------------------------------------------------------------------
-
-    /// Constructor
-    CUB_RUNTIME_FUNCTION __forceinline__
-    DispatchSegmentedReduce(
-        void*                   d_temp_storage,
-        size_t                  &temp_storage_bytes,
-        InputIteratorT          d_in,
-        OutputIteratorT         d_out,
-        OffsetT                 num_segments,
-        OffsetIteratorT         d_begin_offsets,
-        OffsetIteratorT         d_end_offsets,
-        ReductionOpT            reduction_op,
-        OutputT                 init,
-        cudaStream_t            stream,
-        bool                    debug_synchronous,
-        int                     ptx_version)
-    :
-        d_temp_storage(d_temp_storage),
-        temp_storage_bytes(temp_storage_bytes),
-        d_in(d_in),
-        d_out(d_out),
-        num_segments(num_segments),
-        d_begin_offsets(d_begin_offsets),
-        d_end_offsets(d_end_offsets),
-        reduction_op(reduction_op),
-        init(init),
-        stream(stream),
-        debug_synchronous(debug_synchronous),
-        ptx_version(ptx_version)
-    {}
-
-
-
-    //------------------------------------------------------------------------------
-    // Chained policy invocation
-    //------------------------------------------------------------------------------
-
-    /// Invocation
-    template <
-        typename                        ActivePolicyT,                  ///< Umbrella policy active for the target device
-        typename                        DeviceSegmentedReduceKernelT>   ///< Function type of cub::DeviceSegmentedReduceKernel
-    CUB_RUNTIME_FUNCTION __forceinline__
-    cudaError_t InvokePasses(
-        DeviceSegmentedReduceKernelT    segmented_reduce_kernel)        ///< [in] Kernel function pointer to parameterization of cub::DeviceSegmentedReduceKernel
-    {
-#ifndef CUB_RUNTIME_ENABLED
-        (void)segmented_reduce_kernel;
-        // Kernel launch not supported from this device
-        return CubDebug(cudaErrorNotSupported );
-#else
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Return if the caller is simply requesting the size of the storage allocation
-            if (d_temp_storage == NULL)
-            {
-                temp_storage_bytes = 1;
-                return cudaSuccess;
-            }
-
-            // Init kernel configuration
-            KernelConfig segmented_reduce_config;
-            if (CubDebug(error = segmented_reduce_config.Init<typename ActivePolicyT::SegmentedReducePolicy>(segmented_reduce_kernel))) break;
-
-            // Log device_reduce_sweep_kernel configuration
-            if (debug_synchronous) _CubLog("Invoking SegmentedDeviceReduceKernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
-                num_segments,
-                ActivePolicyT::SegmentedReducePolicy::BLOCK_THREADS,
-                (long long) stream,
-                ActivePolicyT::SegmentedReducePolicy::ITEMS_PER_THREAD,
-                segmented_reduce_config.sm_occupancy);
-
-            // Invoke DeviceReduceKernel
-            segmented_reduce_kernel<<<num_segments, ActivePolicyT::SegmentedReducePolicy::BLOCK_THREADS, 0, stream>>>(
-                d_in,
-                d_out,
-                d_begin_offsets,
-                d_end_offsets,
-                num_segments,
-                reduction_op,
-                init);
-
-            // Check for failure to launch
-            if (CubDebug(error = cudaPeekAtLastError())) break;
-
-            // Sync the stream if specified to flush runtime errors
-            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-        }
-        while (0);
-
-        return error;
-
-#endif // CUB_RUNTIME_ENABLED
-
-    }
-
-
-    /// Invocation
-    template <typename ActivePolicyT>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    cudaError_t Invoke()
-    {
-        typedef typename DispatchSegmentedReduce::MaxPolicy MaxPolicyT;
-
-        // Force kernel code-generation in all compiler passes
-        return InvokePasses<ActivePolicyT>(
-            DeviceSegmentedReduceKernel<MaxPolicyT, InputIteratorT, OutputIteratorT, OffsetIteratorT, OffsetT, ReductionOpT, OutputT>);
-    }
-
-
-    //------------------------------------------------------------------------------
-    // Dispatch entrypoints
-    //------------------------------------------------------------------------------
-
-    /**
-     * Internal dispatch routine for computing a device-wide reduction
-     */
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Dispatch(
-        void            *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t          &temp_storage_bytes,                ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT  d_in,                               ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT d_out,                              ///< [out] Pointer to the output aggregate
-        int             num_segments,                       ///< [in] The number of segments that comprise the sorting data
-        OffsetIteratorT d_begin_offsets,                    ///< [in] Pointer to the sequence of beginning offsets of length \p num_segments, such that <tt>d_begin_offsets[i]</tt> is the first element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>
-        OffsetIteratorT d_end_offsets,                      ///< [in] Pointer to the sequence of ending offsets of length \p num_segments, such that <tt>d_end_offsets[i]-1</tt> is the last element of the <em>i</em><sup>th</sup> data segment in <tt>d_keys_*</tt> and <tt>d_values_*</tt>.  If <tt>d_end_offsets[i]-1</tt> <= <tt>d_begin_offsets[i]</tt>, the <em>i</em><sup>th</sup> is considered empty.
-        ReductionOpT    reduction_op,                       ///< [in] Binary reduction functor 
-        OutputT         init,                               ///< [in] The initial value of the reduction
-        cudaStream_t    stream,                             ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool            debug_synchronous)                  ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        typedef typename DispatchSegmentedReduce::MaxPolicy MaxPolicyT;
-
-        if (num_segments <= 0)
-            return cudaSuccess;
-
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Get PTX version
-            int ptx_version;
-            if (CubDebug(error = PtxVersion(ptx_version))) break;
-
-            // Create dispatch functor
-            DispatchSegmentedReduce dispatch(
-                d_temp_storage, temp_storage_bytes,
-                d_in, d_out,
-                num_segments, d_begin_offsets, d_end_offsets,
-                reduction_op, init,
-                stream, debug_synchronous, ptx_version);
-
-            // Dispatch to chained policy
-            if (CubDebug(error = MaxPolicyT::Invoke(ptx_version, dispatch))) break;
-        }
-        while (0);
-
-        return error;
-    }
-};
-
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/device/dispatch/dispatch_reduce_by_key.cuh b/pfsimulator/third_party/cub/device/dispatch/dispatch_reduce_by_key.cuh
deleted file mode 100644
index 6f4837b7f..000000000
--- a/pfsimulator/third_party/cub/device/dispatch/dispatch_reduce_by_key.cuh
+++ /dev/null
@@ -1,554 +0,0 @@
-
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::DeviceReduceByKey provides device-wide, parallel operations for reducing segments of values residing within device-accessible memory.
- */
-
-#pragma once
-
-#include <stdio.h>
-#include <iterator>
-
-#include "dispatch_scan.cuh"
-#include "../../agent/agent_reduce_by_key.cuh"
-#include "../../thread/thread_operators.cuh"
-#include "../../grid/grid_queue.cuh"
-#include "../../util_device.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/******************************************************************************
- * Kernel entry points
- *****************************************************************************/
-
-/**
- * Multi-block reduce-by-key sweep kernel entry point
- */
-template <
-    typename            AgentReduceByKeyPolicyT,                 ///< Parameterized AgentReduceByKeyPolicyT tuning policy type
-    typename            KeysInputIteratorT,                     ///< Random-access input iterator type for keys
-    typename            UniqueOutputIteratorT,                  ///< Random-access output iterator type for keys
-    typename            ValuesInputIteratorT,                   ///< Random-access input iterator type for values
-    typename            AggregatesOutputIteratorT,              ///< Random-access output iterator type for values
-    typename            NumRunsOutputIteratorT,                 ///< Output iterator type for recording number of segments encountered
-    typename            ScanTileStateT,                         ///< Tile status interface type
-    typename            EqualityOpT,                            ///< KeyT equality operator type
-    typename            ReductionOpT,                           ///< ValueT reduction operator type
-    typename            OffsetT>                                ///< Signed integer type for global offsets
-__launch_bounds__ (int(AgentReduceByKeyPolicyT::BLOCK_THREADS))
-__global__ void DeviceReduceByKeyKernel(
-    KeysInputIteratorT          d_keys_in,                      ///< Pointer to the input sequence of keys
-    UniqueOutputIteratorT       d_unique_out,                   ///< Pointer to the output sequence of unique keys (one key per run)
-    ValuesInputIteratorT        d_values_in,                    ///< Pointer to the input sequence of corresponding values
-    AggregatesOutputIteratorT   d_aggregates_out,               ///< Pointer to the output sequence of value aggregates (one aggregate per run)
-    NumRunsOutputIteratorT      d_num_runs_out,                 ///< Pointer to total number of runs encountered (i.e., the length of d_unique_out)
-    ScanTileStateT              tile_state,                     ///< Tile status interface
-    int                         start_tile,                     ///< The starting tile for the current grid
-    EqualityOpT                 equality_op,                    ///< KeyT equality operator
-    ReductionOpT                reduction_op,                   ///< ValueT reduction operator
-    OffsetT                     num_items)                      ///< Total number of items to select from
-{
-    // Thread block type for reducing tiles of value segments
-    typedef AgentReduceByKey<
-            AgentReduceByKeyPolicyT,
-            KeysInputIteratorT,
-            UniqueOutputIteratorT,
-            ValuesInputIteratorT,
-            AggregatesOutputIteratorT,
-            NumRunsOutputIteratorT,
-            EqualityOpT,
-            ReductionOpT,
-            OffsetT>
-        AgentReduceByKeyT;
-
-    // Shared memory for AgentReduceByKey
-    __shared__ typename AgentReduceByKeyT::TempStorage temp_storage;
-
-    // Process tiles
-    AgentReduceByKeyT(temp_storage, d_keys_in, d_unique_out, d_values_in, d_aggregates_out, d_num_runs_out, equality_op, reduction_op).ConsumeRange(
-        num_items,
-        tile_state,
-        start_tile);
-}
-
-
-
-
-/******************************************************************************
- * Dispatch
- ******************************************************************************/
-
-/**
- * Utility class for dispatching the appropriately-tuned kernels for DeviceReduceByKey
- */
-template <
-    typename    KeysInputIteratorT,         ///< Random-access input iterator type for keys
-    typename    UniqueOutputIteratorT,      ///< Random-access output iterator type for keys
-    typename    ValuesInputIteratorT,       ///< Random-access input iterator type for values
-    typename    AggregatesOutputIteratorT,  ///< Random-access output iterator type for values
-    typename    NumRunsOutputIteratorT,     ///< Output iterator type for recording number of segments encountered
-    typename    EqualityOpT,                ///< KeyT equality operator type
-    typename    ReductionOpT,               ///< ValueT reduction operator type
-    typename    OffsetT>                    ///< Signed integer type for global offsets
-struct DispatchReduceByKey
-{
-    //-------------------------------------------------------------------------
-    // Types and constants
-    //-------------------------------------------------------------------------
-
-    // The input keys type
-    typedef typename std::iterator_traits<KeysInputIteratorT>::value_type KeyInputT;
-
-    // The output keys type
-    typedef typename If<(Equals<typename std::iterator_traits<UniqueOutputIteratorT>::value_type, void>::VALUE),    // KeyOutputT =  (if output iterator's value type is void) ?
-        typename std::iterator_traits<KeysInputIteratorT>::value_type,                                              // ... then the input iterator's value type,
-        typename std::iterator_traits<UniqueOutputIteratorT>::value_type>::Type KeyOutputT;                         // ... else the output iterator's value type
-
-    // The input values type
-    typedef typename std::iterator_traits<ValuesInputIteratorT>::value_type ValueInputT;
-
-    // The output values type
-    typedef typename If<(Equals<typename std::iterator_traits<AggregatesOutputIteratorT>::value_type, void>::VALUE),    // ValueOutputT =  (if output iterator's value type is void) ?
-        typename std::iterator_traits<ValuesInputIteratorT>::value_type,                                                // ... then the input iterator's value type,
-        typename std::iterator_traits<AggregatesOutputIteratorT>::value_type>::Type ValueOutputT;                       // ... else the output iterator's value type
-
-    enum
-    {
-        INIT_KERNEL_THREADS     = 128,
-        MAX_INPUT_BYTES         = CUB_MAX(sizeof(KeyOutputT), sizeof(ValueOutputT)),
-        COMBINED_INPUT_BYTES    = sizeof(KeyOutputT) + sizeof(ValueOutputT),
-    };
-
-    // Tile status descriptor interface type
-    typedef ReduceByKeyScanTileState<ValueOutputT, OffsetT> ScanTileStateT;
-
-
-    //-------------------------------------------------------------------------
-    // Tuning policies
-    //-------------------------------------------------------------------------
-
-    /// SM35
-    struct Policy350
-    {
-        enum {
-            NOMINAL_4B_ITEMS_PER_THREAD = 6,
-            ITEMS_PER_THREAD            = (MAX_INPUT_BYTES <= 8) ? 6 : CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, ((NOMINAL_4B_ITEMS_PER_THREAD * 8) + COMBINED_INPUT_BYTES - 1) / COMBINED_INPUT_BYTES)),
-        };
-
-        typedef AgentReduceByKeyPolicy<
-                128,
-                ITEMS_PER_THREAD,
-                BLOCK_LOAD_DIRECT,
-                LOAD_LDG,
-                BLOCK_SCAN_WARP_SCANS>
-            ReduceByKeyPolicyT;
-    };
-
-    /// SM30
-    struct Policy300
-    {
-        enum {
-            NOMINAL_4B_ITEMS_PER_THREAD = 6,
-            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, ((NOMINAL_4B_ITEMS_PER_THREAD * 8) + COMBINED_INPUT_BYTES - 1) / COMBINED_INPUT_BYTES)),
-        };
-
-        typedef AgentReduceByKeyPolicy<
-                128,
-                ITEMS_PER_THREAD,
-                BLOCK_LOAD_WARP_TRANSPOSE,
-                LOAD_DEFAULT,
-                BLOCK_SCAN_WARP_SCANS>
-            ReduceByKeyPolicyT;
-    };
-
-    /// SM20
-    struct Policy200
-    {
-        enum {
-            NOMINAL_4B_ITEMS_PER_THREAD = 11,
-            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, ((NOMINAL_4B_ITEMS_PER_THREAD * 8) + COMBINED_INPUT_BYTES - 1) / COMBINED_INPUT_BYTES)),
-        };
-
-        typedef AgentReduceByKeyPolicy<
-                128,
-                ITEMS_PER_THREAD,
-                BLOCK_LOAD_WARP_TRANSPOSE,
-                LOAD_DEFAULT,
-                BLOCK_SCAN_WARP_SCANS>
-            ReduceByKeyPolicyT;
-    };
-
-    /// SM13
-    struct Policy130
-    {
-        enum {
-            NOMINAL_4B_ITEMS_PER_THREAD = 7,
-            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, ((NOMINAL_4B_ITEMS_PER_THREAD * 8) + COMBINED_INPUT_BYTES - 1) / COMBINED_INPUT_BYTES)),
-        };
-
-        typedef AgentReduceByKeyPolicy<
-                128,
-                ITEMS_PER_THREAD,
-                BLOCK_LOAD_WARP_TRANSPOSE,
-                LOAD_DEFAULT,
-                BLOCK_SCAN_WARP_SCANS>
-            ReduceByKeyPolicyT;
-    };
-
-    /// SM11
-    struct Policy110
-    {
-        enum {
-            NOMINAL_4B_ITEMS_PER_THREAD = 5,
-            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 8) / COMBINED_INPUT_BYTES)),
-        };
-
-        typedef AgentReduceByKeyPolicy<
-                64,
-                ITEMS_PER_THREAD,
-                BLOCK_LOAD_WARP_TRANSPOSE,
-                LOAD_DEFAULT,
-                BLOCK_SCAN_RAKING>
-            ReduceByKeyPolicyT;
-    };
-
-
-    /******************************************************************************
-     * Tuning policies of current PTX compiler pass
-     ******************************************************************************/
-
-#if (CUB_PTX_ARCH >= 350)
-    typedef Policy350 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 300)
-    typedef Policy300 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 200)
-    typedef Policy200 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 130)
-    typedef Policy130 PtxPolicy;
-
-#else
-    typedef Policy110 PtxPolicy;
-
-#endif
-
-    // "Opaque" policies (whose parameterizations aren't reflected in the type signature)
-    struct PtxReduceByKeyPolicy : PtxPolicy::ReduceByKeyPolicyT {};
-
-
-    /******************************************************************************
-     * Utilities
-     ******************************************************************************/
-
-    /**
-     * Initialize kernel dispatch configurations with the policies corresponding to the PTX assembly we will use
-     */
-    template <typename KernelConfig>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static void InitConfigs(
-        int             ptx_version,
-        KernelConfig    &reduce_by_key_config)
-    {
-    #if (CUB_PTX_ARCH > 0)
-        (void)ptx_version;
-
-        // We're on the device, so initialize the kernel dispatch configurations with the current PTX policy
-        reduce_by_key_config.template Init<PtxReduceByKeyPolicy>();
-
-    #else
-
-        // We're on the host, so lookup and initialize the kernel dispatch configurations with the policies that match the device's PTX version
-        if (ptx_version >= 350)
-        {
-            reduce_by_key_config.template Init<typename Policy350::ReduceByKeyPolicyT>();
-        }
-        else if (ptx_version >= 300)
-        {
-            reduce_by_key_config.template Init<typename Policy300::ReduceByKeyPolicyT>();
-        }
-        else if (ptx_version >= 200)
-        {
-            reduce_by_key_config.template Init<typename Policy200::ReduceByKeyPolicyT>();
-        }
-        else if (ptx_version >= 130)
-        {
-            reduce_by_key_config.template Init<typename Policy130::ReduceByKeyPolicyT>();
-        }
-        else
-        {
-            reduce_by_key_config.template Init<typename Policy110::ReduceByKeyPolicyT>();
-        }
-
-    #endif
-    }
-
-
-    /**
-     * Kernel kernel dispatch configuration.
-     */
-    struct KernelConfig
-    {
-        int block_threads;
-        int items_per_thread;
-        int tile_items;
-
-        template <typename PolicyT>
-        CUB_RUNTIME_FUNCTION __forceinline__
-        void Init()
-        {
-            block_threads       = PolicyT::BLOCK_THREADS;
-            items_per_thread    = PolicyT::ITEMS_PER_THREAD;
-            tile_items          = block_threads * items_per_thread;
-        }
-    };
-
-
-    //---------------------------------------------------------------------
-    // Dispatch entrypoints
-    //---------------------------------------------------------------------
-
-    /**
-     * Internal dispatch routine for computing a device-wide reduce-by-key using the
-     * specified kernel functions.
-     */
-    template <
-        typename                    ScanInitKernelT,         ///< Function type of cub::DeviceScanInitKernel
-        typename                    ReduceByKeyKernelT>      ///< Function type of cub::DeviceReduceByKeyKernelT
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Dispatch(
-        void*                       d_temp_storage,             ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&                     temp_storage_bytes,         ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        KeysInputIteratorT          d_keys_in,                  ///< [in] Pointer to the input sequence of keys
-        UniqueOutputIteratorT       d_unique_out,               ///< [out] Pointer to the output sequence of unique keys (one key per run)
-        ValuesInputIteratorT        d_values_in,                ///< [in] Pointer to the input sequence of corresponding values
-        AggregatesOutputIteratorT   d_aggregates_out,           ///< [out] Pointer to the output sequence of value aggregates (one aggregate per run)
-        NumRunsOutputIteratorT      d_num_runs_out,             ///< [out] Pointer to total number of runs encountered (i.e., the length of d_unique_out)
-        EqualityOpT                 equality_op,                ///< [in] KeyT equality operator
-        ReductionOpT                reduction_op,               ///< [in] ValueT reduction operator
-        OffsetT                     num_items,                  ///< [in] Total number of items to select from
-        cudaStream_t                stream,                     ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous,          ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-        int                         /*ptx_version*/,            ///< [in] PTX version of dispatch kernels
-        ScanInitKernelT                init_kernel,                ///< [in] Kernel function pointer to parameterization of cub::DeviceScanInitKernel
-        ReduceByKeyKernelT             reduce_by_key_kernel,       ///< [in] Kernel function pointer to parameterization of cub::DeviceReduceByKeyKernel
-        KernelConfig                reduce_by_key_config)       ///< [in] Dispatch parameters that match the policy that \p reduce_by_key_kernel was compiled for
-    {
-
-#ifndef CUB_RUNTIME_ENABLED
-      (void)d_temp_storage;
-      (void)temp_storage_bytes;
-      (void)d_keys_in;
-      (void)d_unique_out;
-      (void)d_values_in;
-      (void)d_aggregates_out;
-      (void)d_num_runs_out;
-      (void)equality_op;
-      (void)reduction_op;
-      (void)num_items;
-      (void)stream;
-      (void)debug_synchronous;
-      (void)init_kernel;
-      (void)reduce_by_key_kernel;
-      (void)reduce_by_key_config;
-
-        // Kernel launch not supported from this device
-        return CubDebug(cudaErrorNotSupported);
-
-#else
-
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Get device ordinal
-            int device_ordinal;
-            if (CubDebug(error = cudaGetDevice(&device_ordinal))) break;
-
-            // Get SM count
-            int sm_count;
-            if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break;
-
-            // Number of input tiles
-            int tile_size = reduce_by_key_config.block_threads * reduce_by_key_config.items_per_thread;
-            int num_tiles = (num_items + tile_size - 1) / tile_size;
-
-            // Specify temporary storage allocation requirements
-            size_t  allocation_sizes[1];
-            if (CubDebug(error = ScanTileStateT::AllocationSize(num_tiles, allocation_sizes[0]))) break;    // bytes needed for tile status descriptors
-
-            // Compute allocation pointers into the single storage blob (or compute the necessary size of the blob)
-            void* allocations[1];
-            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
-            if (d_temp_storage == NULL)
-            {
-                // Return if the caller is simply requesting the size of the storage allocation
-                break;
-            }
-
-            // Construct the tile status interface
-            ScanTileStateT tile_state;
-            if (CubDebug(error = tile_state.Init(num_tiles, allocations[0], allocation_sizes[0]))) break;
-
-            // Log init_kernel configuration
-            int init_grid_size = CUB_MAX(1, (num_tiles + INIT_KERNEL_THREADS - 1) / INIT_KERNEL_THREADS);
-            if (debug_synchronous) _CubLog("Invoking init_kernel<<<%d, %d, 0, %lld>>>()\n", init_grid_size, INIT_KERNEL_THREADS, (long long) stream);
-
-            // Invoke init_kernel to initialize tile descriptors
-            init_kernel<<<init_grid_size, INIT_KERNEL_THREADS, 0, stream>>>(
-                tile_state,
-                num_tiles,
-                d_num_runs_out);
-
-            // Check for failure to launch
-            if (CubDebug(error = cudaPeekAtLastError())) break;
-
-            // Sync the stream if specified to flush runtime errors
-            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-
-            // Return if empty problem
-            if (num_items == 0)
-                break;
-
-            // Get SM occupancy for reduce_by_key_kernel
-            int reduce_by_key_sm_occupancy;
-            if (CubDebug(error = MaxSmOccupancy(
-                reduce_by_key_sm_occupancy,            // out
-                reduce_by_key_kernel,
-                reduce_by_key_config.block_threads))) break;
-
-            // Get max x-dimension of grid
-            int max_dim_x;
-            if (CubDebug(error = cudaDeviceGetAttribute(&max_dim_x, cudaDevAttrMaxGridDimX, device_ordinal))) break;;
-
-            // Run grids in epochs (in case number of tiles exceeds max x-dimension
-            int scan_grid_size = CUB_MIN(num_tiles, max_dim_x);
-            for (int start_tile = 0; start_tile < num_tiles; start_tile += scan_grid_size)
-            {
-                // Log reduce_by_key_kernel configuration
-                if (debug_synchronous) _CubLog("Invoking %d reduce_by_key_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
-                    start_tile, scan_grid_size, reduce_by_key_config.block_threads, (long long) stream, reduce_by_key_config.items_per_thread, reduce_by_key_sm_occupancy);
-
-                // Invoke reduce_by_key_kernel
-                reduce_by_key_kernel<<<scan_grid_size, reduce_by_key_config.block_threads, 0, stream>>>(
-                    d_keys_in,
-                    d_unique_out,
-                    d_values_in,
-                    d_aggregates_out,
-                    d_num_runs_out,
-                    tile_state,
-                    start_tile,
-                    equality_op,
-                    reduction_op,
-                    num_items);
-
-                // Check for failure to launch
-                if (CubDebug(error = cudaPeekAtLastError())) break;
-
-                // Sync the stream if specified to flush runtime errors
-                if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-            }
-        }
-        while (0);
-
-        return error;
-
-#endif  // CUB_RUNTIME_ENABLED
-    }
-
-
-    /**
-     * Internal dispatch routine
-     */
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Dispatch(
-        void*                       d_temp_storage,                 ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&                     temp_storage_bytes,             ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        KeysInputIteratorT          d_keys_in,                      ///< [in] Pointer to the input sequence of keys
-        UniqueOutputIteratorT       d_unique_out,                   ///< [out] Pointer to the output sequence of unique keys (one key per run)
-        ValuesInputIteratorT        d_values_in,                    ///< [in] Pointer to the input sequence of corresponding values
-        AggregatesOutputIteratorT   d_aggregates_out,               ///< [out] Pointer to the output sequence of value aggregates (one aggregate per run)
-        NumRunsOutputIteratorT      d_num_runs_out,                 ///< [out] Pointer to total number of runs encountered (i.e., the length of d_unique_out)
-        EqualityOpT                 equality_op,                    ///< [in] KeyT equality operator
-        ReductionOpT                reduction_op,                   ///< [in] ValueT reduction operator
-        OffsetT                     num_items,                      ///< [in] Total number of items to select from
-        cudaStream_t                stream,                         ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous)              ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Get PTX version
-            int ptx_version;
-    #if (CUB_PTX_ARCH == 0)
-            if (CubDebug(error = PtxVersion(ptx_version))) break;
-    #else
-            ptx_version = CUB_PTX_ARCH;
-    #endif
-
-            // Get kernel kernel dispatch configurations
-            KernelConfig reduce_by_key_config;
-            InitConfigs(ptx_version, reduce_by_key_config);
-
-            // Dispatch
-            if (CubDebug(error = Dispatch(
-                d_temp_storage,
-                temp_storage_bytes,
-                d_keys_in,
-                d_unique_out,
-                d_values_in,
-                d_aggregates_out,
-                d_num_runs_out,
-                equality_op,
-                reduction_op,
-                num_items,
-                stream,
-                debug_synchronous,
-                ptx_version,
-                DeviceCompactInitKernel<ScanTileStateT, NumRunsOutputIteratorT>,
-                DeviceReduceByKeyKernel<PtxReduceByKeyPolicy, KeysInputIteratorT, UniqueOutputIteratorT, ValuesInputIteratorT, AggregatesOutputIteratorT, NumRunsOutputIteratorT, ScanTileStateT, EqualityOpT, ReductionOpT, OffsetT>,
-                reduce_by_key_config))) break;
-        }
-        while (0);
-
-        return error;
-    }
-};
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/device/dispatch/dispatch_rle.cuh b/pfsimulator/third_party/cub/device/dispatch/dispatch_rle.cuh
deleted file mode 100644
index 98c3681f0..000000000
--- a/pfsimulator/third_party/cub/device/dispatch/dispatch_rle.cuh
+++ /dev/null
@@ -1,538 +0,0 @@
-
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::DeviceRle provides device-wide, parallel operations for run-length-encoding sequences of data items residing within device-accessible memory.
- */
-
-#pragma once
-
-#include <stdio.h>
-#include <iterator>
-
-#include "dispatch_scan.cuh"
-#include "../../agent/agent_rle.cuh"
-#include "../../thread/thread_operators.cuh"
-#include "../../grid/grid_queue.cuh"
-#include "../../util_device.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/******************************************************************************
- * Kernel entry points
- *****************************************************************************/
-
-/**
- * Select kernel entry point (multi-block)
- *
- * Performs functor-based selection if SelectOp functor type != NullType
- * Otherwise performs flag-based selection if FlagIterator's value type != NullType
- * Otherwise performs discontinuity selection (keep unique)
- */
-template <
-    typename            AgentRlePolicyT,        ///< Parameterized AgentRlePolicyT tuning policy type
-    typename            InputIteratorT,             ///< Random-access input iterator type for reading input items \iterator
-    typename            OffsetsOutputIteratorT,     ///< Random-access output iterator type for writing run-offset values \iterator
-    typename            LengthsOutputIteratorT,     ///< Random-access output iterator type for writing run-length values \iterator
-    typename            NumRunsOutputIteratorT,     ///< Output iterator type for recording the number of runs encountered \iterator
-    typename            ScanTileStateT,              ///< Tile status interface type
-    typename            EqualityOpT,                 ///< T equality operator type
-    typename            OffsetT>                    ///< Signed integer type for global offsets
-__launch_bounds__ (int(AgentRlePolicyT::BLOCK_THREADS))
-__global__ void DeviceRleSweepKernel(
-    InputIteratorT              d_in,               ///< [in] Pointer to input sequence of data items
-    OffsetsOutputIteratorT      d_offsets_out,      ///< [out] Pointer to output sequence of run-offsets
-    LengthsOutputIteratorT      d_lengths_out,      ///< [out] Pointer to output sequence of run-lengths
-    NumRunsOutputIteratorT      d_num_runs_out,     ///< [out] Pointer to total number of runs (i.e., length of \p d_offsets_out)
-    ScanTileStateT              tile_status,        ///< [in] Tile status interface
-    EqualityOpT                 equality_op,        ///< [in] Equality operator for input items
-    OffsetT                     num_items,          ///< [in] Total number of input items (i.e., length of \p d_in)
-    int                         num_tiles)          ///< [in] Total number of tiles for the entire problem
-{
-    // Thread block type for selecting data from input tiles
-    typedef AgentRle<
-        AgentRlePolicyT,
-        InputIteratorT,
-        OffsetsOutputIteratorT,
-        LengthsOutputIteratorT,
-        EqualityOpT,
-        OffsetT> AgentRleT;
-
-    // Shared memory for AgentRle
-    __shared__ typename AgentRleT::TempStorage temp_storage;
-
-    // Process tiles
-    AgentRleT(temp_storage, d_in, d_offsets_out, d_lengths_out, equality_op, num_items).ConsumeRange(
-        num_tiles,
-        tile_status,
-        d_num_runs_out);
-}
-
-
-
-
-/******************************************************************************
- * Dispatch
- ******************************************************************************/
-
-/**
- * Utility class for dispatching the appropriately-tuned kernels for DeviceRle
- */
-template <
-    typename            InputIteratorT,             ///< Random-access input iterator type for reading input items \iterator
-    typename            OffsetsOutputIteratorT,     ///< Random-access output iterator type for writing run-offset values \iterator
-    typename            LengthsOutputIteratorT,     ///< Random-access output iterator type for writing run-length values \iterator
-    typename            NumRunsOutputIteratorT,     ///< Output iterator type for recording the number of runs encountered \iterator
-    typename            EqualityOpT,                ///< T equality operator type
-    typename            OffsetT>                    ///< Signed integer type for global offsets
-struct DeviceRleDispatch
-{
-    /******************************************************************************
-     * Types and constants
-     ******************************************************************************/
-
-    // The input value type
-    typedef typename std::iterator_traits<InputIteratorT>::value_type T;
-
-    // The lengths output value type
-    typedef typename If<(Equals<typename std::iterator_traits<LengthsOutputIteratorT>::value_type, void>::VALUE),   // LengthT =  (if output iterator's value type is void) ?
-        OffsetT,                                                                                                    // ... then the OffsetT type,
-        typename std::iterator_traits<LengthsOutputIteratorT>::value_type>::Type LengthT;                           // ... else the output iterator's value type
-
-    enum
-    {
-        INIT_KERNEL_THREADS = 128,
-    };
-
-    // Tile status descriptor interface type
-    typedef ReduceByKeyScanTileState<LengthT, OffsetT> ScanTileStateT;
-
-
-    /******************************************************************************
-     * Tuning policies
-     ******************************************************************************/
-
-    /// SM35
-    struct Policy350
-    {
-        enum {
-            NOMINAL_4B_ITEMS_PER_THREAD = 15,
-            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(T)))),
-        };
-
-        typedef AgentRlePolicy<
-                96,
-                ITEMS_PER_THREAD,
-                BLOCK_LOAD_DIRECT,
-                LOAD_LDG,
-                true,
-                BLOCK_SCAN_WARP_SCANS>
-            RleSweepPolicy;
-    };
-
-    /// SM30
-    struct Policy300
-    {
-        enum {
-            NOMINAL_4B_ITEMS_PER_THREAD = 5,
-            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(T)))),
-        };
-
-        typedef AgentRlePolicy<
-                256,
-                ITEMS_PER_THREAD,
-                BLOCK_LOAD_WARP_TRANSPOSE,
-                LOAD_DEFAULT,
-                true,
-                BLOCK_SCAN_RAKING_MEMOIZE>
-            RleSweepPolicy;
-    };
-
-    /// SM20
-    struct Policy200
-    {
-        enum {
-            NOMINAL_4B_ITEMS_PER_THREAD = 15,
-            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(T)))),
-        };
-
-        typedef AgentRlePolicy<
-                128,
-                ITEMS_PER_THREAD,
-                BLOCK_LOAD_WARP_TRANSPOSE,
-                LOAD_DEFAULT,
-                false,
-                BLOCK_SCAN_WARP_SCANS>
-            RleSweepPolicy;
-    };
-
-    /// SM13
-    struct Policy130
-    {
-        enum {
-            NOMINAL_4B_ITEMS_PER_THREAD = 9,
-            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(T)))),
-        };
-
-        typedef AgentRlePolicy<
-                64,
-                ITEMS_PER_THREAD,
-                BLOCK_LOAD_WARP_TRANSPOSE,
-                LOAD_DEFAULT,
-                true,
-                BLOCK_SCAN_RAKING_MEMOIZE>
-            RleSweepPolicy;
-    };
-
-    /// SM10
-    struct Policy100
-    {
-        enum {
-            NOMINAL_4B_ITEMS_PER_THREAD = 9,
-            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(T)))),
-        };
-
-        typedef AgentRlePolicy<
-                256,
-                ITEMS_PER_THREAD,
-                BLOCK_LOAD_WARP_TRANSPOSE,
-                LOAD_DEFAULT,
-                true,
-                BLOCK_SCAN_RAKING_MEMOIZE>
-            RleSweepPolicy;
-    };
-
-
-    /******************************************************************************
-     * Tuning policies of current PTX compiler pass
-     ******************************************************************************/
-
-#if (CUB_PTX_ARCH >= 350)
-    typedef Policy350 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 300)
-    typedef Policy300 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 200)
-    typedef Policy200 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 130)
-    typedef Policy130 PtxPolicy;
-
-#else
-    typedef Policy100 PtxPolicy;
-
-#endif
-
-    // "Opaque" policies (whose parameterizations aren't reflected in the type signature)
-    struct PtxRleSweepPolicy : PtxPolicy::RleSweepPolicy {};
-
-
-    /******************************************************************************
-     * Utilities
-     ******************************************************************************/
-
-    /**
-     * Initialize kernel dispatch configurations with the policies corresponding to the PTX assembly we will use
-     */
-    template <typename KernelConfig>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static void InitConfigs(
-        int             ptx_version,
-        KernelConfig&   device_rle_config)
-    {
-    #if (CUB_PTX_ARCH > 0)
-
-        // We're on the device, so initialize the kernel dispatch configurations with the current PTX policy
-        device_rle_config.template Init<PtxRleSweepPolicy>();
-
-    #else
-
-        // We're on the host, so lookup and initialize the kernel dispatch configurations with the policies that match the device's PTX version
-        if (ptx_version >= 350)
-        {
-            device_rle_config.template Init<typename Policy350::RleSweepPolicy>();
-        }
-        else if (ptx_version >= 300)
-        {
-            device_rle_config.template Init<typename Policy300::RleSweepPolicy>();
-        }
-        else if (ptx_version >= 200)
-        {
-            device_rle_config.template Init<typename Policy200::RleSweepPolicy>();
-        }
-        else if (ptx_version >= 130)
-        {
-            device_rle_config.template Init<typename Policy130::RleSweepPolicy>();
-        }
-        else
-        {
-            device_rle_config.template Init<typename Policy100::RleSweepPolicy>();
-        }
-
-    #endif
-    }
-
-
-    /**
-     * Kernel kernel dispatch configuration.  Mirrors the constants within AgentRlePolicyT.
-     */
-    struct KernelConfig
-    {
-        int                     block_threads;
-        int                     items_per_thread;
-        BlockLoadAlgorithm      load_policy;
-        bool                    store_warp_time_slicing;
-        BlockScanAlgorithm      scan_algorithm;
-
-        template <typename AgentRlePolicyT>
-        CUB_RUNTIME_FUNCTION __forceinline__
-        void Init()
-        {
-            block_threads               = AgentRlePolicyT::BLOCK_THREADS;
-            items_per_thread            = AgentRlePolicyT::ITEMS_PER_THREAD;
-            load_policy                 = AgentRlePolicyT::LOAD_ALGORITHM;
-            store_warp_time_slicing     = AgentRlePolicyT::STORE_WARP_TIME_SLICING;
-            scan_algorithm              = AgentRlePolicyT::SCAN_ALGORITHM;
-        }
-
-        CUB_RUNTIME_FUNCTION __forceinline__
-        void Print()
-        {
-            printf("%d, %d, %d, %d, %d",
-                block_threads,
-                items_per_thread,
-                load_policy,
-                store_warp_time_slicing,
-                scan_algorithm);
-        }
-    };
-
-
-    /******************************************************************************
-     * Dispatch entrypoints
-     ******************************************************************************/
-
-    /**
-     * Internal dispatch routine for computing a device-wide run-length-encode using the
-     * specified kernel functions.
-     */
-    template <
-        typename                    DeviceScanInitKernelPtr,        ///< Function type of cub::DeviceScanInitKernel
-        typename                    DeviceRleSweepKernelPtr>        ///< Function type of cub::DeviceRleSweepKernelPtr
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Dispatch(
-        void*                       d_temp_storage,                 ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&                     temp_storage_bytes,             ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT              d_in,                           ///< [in] Pointer to the input sequence of data items
-        OffsetsOutputIteratorT      d_offsets_out,                  ///< [out] Pointer to the output sequence of run-offsets
-        LengthsOutputIteratorT      d_lengths_out,                  ///< [out] Pointer to the output sequence of run-lengths
-        NumRunsOutputIteratorT      d_num_runs_out,                 ///< [out] Pointer to the total number of runs encountered (i.e., length of \p d_offsets_out)
-        EqualityOpT                 equality_op,                    ///< [in] Equality operator for input items
-        OffsetT                     num_items,                      ///< [in] Total number of input items (i.e., length of \p d_in)
-        cudaStream_t                stream,                         ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous,              ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-        int                         ptx_version,                    ///< [in] PTX version of dispatch kernels
-        DeviceScanInitKernelPtr     device_scan_init_kernel,        ///< [in] Kernel function pointer to parameterization of cub::DeviceScanInitKernel
-        DeviceRleSweepKernelPtr     device_rle_sweep_kernel,        ///< [in] Kernel function pointer to parameterization of cub::DeviceRleSweepKernel
-        KernelConfig                device_rle_config)              ///< [in] Dispatch parameters that match the policy that \p device_rle_sweep_kernel was compiled for
-    {
-
-#ifndef CUB_RUNTIME_ENABLED
-
-        // Kernel launch not supported from this device
-        return CubDebug(cudaErrorNotSupported);
-
-#else
-
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Get device ordinal
-            int device_ordinal;
-            if (CubDebug(error = cudaGetDevice(&device_ordinal))) break;
-
-            // Get SM count
-            int sm_count;
-            if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break;
-
-            // Number of input tiles
-            int tile_size = device_rle_config.block_threads * device_rle_config.items_per_thread;
-            int num_tiles = (num_items + tile_size - 1) / tile_size;
-
-            // Specify temporary storage allocation requirements
-            size_t  allocation_sizes[1];
-            if (CubDebug(error = ScanTileStateT::AllocationSize(num_tiles, allocation_sizes[0]))) break;    // bytes needed for tile status descriptors
-
-            // Compute allocation pointers into the single storage blob (or compute the necessary size of the blob)
-            void* allocations[1];
-            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
-            if (d_temp_storage == NULL)
-            {
-                // Return if the caller is simply requesting the size of the storage allocation
-                break;
-            }
-
-            // Construct the tile status interface
-            ScanTileStateT tile_status;
-            if (CubDebug(error = tile_status.Init(num_tiles, allocations[0], allocation_sizes[0]))) break;
-
-            // Log device_scan_init_kernel configuration
-            int init_grid_size = CUB_MAX(1, (num_tiles + INIT_KERNEL_THREADS - 1) / INIT_KERNEL_THREADS);
-            if (debug_synchronous) _CubLog("Invoking device_scan_init_kernel<<<%d, %d, 0, %lld>>>()\n", init_grid_size, INIT_KERNEL_THREADS, (long long) stream);
-
-            // Invoke device_scan_init_kernel to initialize tile descriptors and queue descriptors
-            device_scan_init_kernel<<<init_grid_size, INIT_KERNEL_THREADS, 0, stream>>>(
-                tile_status,
-                num_tiles,
-                d_num_runs_out);
-
-            // Check for failure to launch
-            if (CubDebug(error = cudaPeekAtLastError())) break;
-
-            // Sync the stream if specified to flush runtime errors
-            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-
-            // Return if empty problem
-            if (num_items == 0)
-                break;
-
-            // Get SM occupancy for device_rle_sweep_kernel
-            int device_rle_kernel_sm_occupancy;
-            if (CubDebug(error = MaxSmOccupancy(
-                device_rle_kernel_sm_occupancy,            // out
-                device_rle_sweep_kernel,
-                device_rle_config.block_threads))) break;
-
-            // Get max x-dimension of grid
-            int max_dim_x;
-            if (CubDebug(error = cudaDeviceGetAttribute(&max_dim_x, cudaDevAttrMaxGridDimX, device_ordinal))) break;;
-
-            // Get grid size for scanning tiles
-            dim3 scan_grid_size;
-            scan_grid_size.z = 1;
-            scan_grid_size.y = ((unsigned int) num_tiles + max_dim_x - 1) / max_dim_x;
-            scan_grid_size.x = CUB_MIN(num_tiles, max_dim_x);
-
-            // Log device_rle_sweep_kernel configuration
-            if (debug_synchronous) _CubLog("Invoking device_rle_sweep_kernel<<<{%d,%d,%d}, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
-                scan_grid_size.x, scan_grid_size.y, scan_grid_size.z, device_rle_config.block_threads, (long long) stream, device_rle_config.items_per_thread, device_rle_kernel_sm_occupancy);
-
-            // Invoke device_rle_sweep_kernel
-            device_rle_sweep_kernel<<<scan_grid_size, device_rle_config.block_threads, 0, stream>>>(
-                d_in,
-                d_offsets_out,
-                d_lengths_out,
-                d_num_runs_out,
-                tile_status,
-                equality_op,
-                num_items,
-                num_tiles);
-
-            // Check for failure to launch
-            if (CubDebug(error = cudaPeekAtLastError())) break;
-
-            // Sync the stream if specified to flush runtime errors
-            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-
-        }
-        while (0);
-
-        return error;
-
-#endif  // CUB_RUNTIME_ENABLED
-    }
-
-
-    /**
-     * Internal dispatch routine
-     */
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Dispatch(
-        void*                       d_temp_storage,                 ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&                     temp_storage_bytes,             ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT              d_in,                           ///< [in] Pointer to input sequence of data items
-        OffsetsOutputIteratorT      d_offsets_out,                  ///< [out] Pointer to output sequence of run-offsets
-        LengthsOutputIteratorT      d_lengths_out,                  ///< [out] Pointer to output sequence of run-lengths
-        NumRunsOutputIteratorT      d_num_runs_out,                 ///< [out] Pointer to total number of runs (i.e., length of \p d_offsets_out)
-        EqualityOpT                 equality_op,                    ///< [in] Equality operator for input items
-        OffsetT                     num_items,                      ///< [in] Total number of input items (i.e., length of \p d_in)
-        cudaStream_t                stream,                         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous)              ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Get PTX version
-            int ptx_version;
-    #if (CUB_PTX_ARCH == 0)
-            if (CubDebug(error = PtxVersion(ptx_version))) break;
-    #else
-            ptx_version = CUB_PTX_ARCH;
-    #endif
-
-            // Get kernel kernel dispatch configurations
-            KernelConfig device_rle_config;
-            InitConfigs(ptx_version, device_rle_config);
-
-            // Dispatch
-            if (CubDebug(error = Dispatch(
-                d_temp_storage,
-                temp_storage_bytes,
-                d_in,
-                d_offsets_out,
-                d_lengths_out,
-                d_num_runs_out,
-                equality_op,
-                num_items,
-                stream,
-                debug_synchronous,
-                ptx_version,
-                DeviceCompactInitKernel<ScanTileStateT, NumRunsOutputIteratorT>,
-                DeviceRleSweepKernel<PtxRleSweepPolicy, InputIteratorT, OffsetsOutputIteratorT, LengthsOutputIteratorT, NumRunsOutputIteratorT, ScanTileStateT, EqualityOpT, OffsetT>,
-                device_rle_config))) break;
-        }
-        while (0);
-
-        return error;
-    }
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/device/dispatch/dispatch_scan.cuh b/pfsimulator/third_party/cub/device/dispatch/dispatch_scan.cuh
deleted file mode 100644
index 3ef720a44..000000000
--- a/pfsimulator/third_party/cub/device/dispatch/dispatch_scan.cuh
+++ /dev/null
@@ -1,563 +0,0 @@
-
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::DeviceScan provides device-wide, parallel operations for computing a prefix scan across a sequence of data items residing within device-accessible memory.
- */
-
-#pragma once
-
-#include <stdio.h>
-#include <iterator>
-
-#include "../../agent/agent_scan.cuh"
-#include "../../thread/thread_operators.cuh"
-#include "../../grid/grid_queue.cuh"
-#include "../../util_arch.cuh"
-#include "../../util_debug.cuh"
-#include "../../util_device.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/******************************************************************************
- * Kernel entry points
- *****************************************************************************/
-
-/**
- * Initialization kernel for tile status initialization (multi-block)
- */
-template <
-    typename            ScanTileStateT>     ///< Tile status interface type
-__global__ void DeviceScanInitKernel(
-    ScanTileStateT      tile_state,         ///< [in] Tile status interface
-    int                 num_tiles)          ///< [in] Number of tiles
-{
-    // Initialize tile status
-    tile_state.InitializeStatus(num_tiles);
-}
-
-/**
- * Initialization kernel for tile status initialization (multi-block)
- */
-template <
-    typename                ScanTileStateT,         ///< Tile status interface type
-    typename                NumSelectedIteratorT>   ///< Output iterator type for recording the number of items selected
-__global__ void DeviceCompactInitKernel(
-    ScanTileStateT          tile_state,             ///< [in] Tile status interface
-    int                     num_tiles,              ///< [in] Number of tiles
-    NumSelectedIteratorT    d_num_selected_out)     ///< [out] Pointer to the total number of items selected (i.e., length of \p d_selected_out)
-{
-    // Initialize tile status
-    tile_state.InitializeStatus(num_tiles);
-
-    // Initialize d_num_selected_out
-    if ((blockIdx.x == 0) && (threadIdx.x == 0))
-        *d_num_selected_out = 0;
-}
-
-
-/**
- * Scan kernel entry point (multi-block)
- */
-template <
-    typename            ScanPolicyT,        ///< Parameterized ScanPolicyT tuning policy type
-    typename            InputIteratorT,     ///< Random-access input iterator type for reading scan inputs \iterator
-    typename            OutputIteratorT,    ///< Random-access output iterator type for writing scan outputs \iterator
-    typename            ScanTileStateT,     ///< Tile status interface type
-    typename            ScanOpT,            ///< Binary scan functor type having member <tt>T operator()(const T &a, const T &b)</tt>
-    typename            InitValueT,         ///< Initial value to seed the exclusive scan (cub::NullType for inclusive scans)
-    typename            OffsetT>            ///< Signed integer type for global offsets
-__launch_bounds__ (int(ScanPolicyT::BLOCK_THREADS))
-__global__ void DeviceScanKernel(
-    InputIteratorT      d_in,               ///< Input data
-    OutputIteratorT     d_out,              ///< Output data
-    ScanTileStateT      tile_state,         ///< Tile status interface
-    int                 start_tile,         ///< The starting tile for the current grid
-    ScanOpT             scan_op,            ///< Binary scan functor 
-    InitValueT          init_value,         ///< Initial value to seed the exclusive scan
-    OffsetT             num_items)          ///< Total number of scan items for the entire problem
-{
-    // Thread block type for scanning input tiles
-    typedef AgentScan<
-        ScanPolicyT,
-        InputIteratorT,
-        OutputIteratorT,
-        ScanOpT,
-        InitValueT,
-        OffsetT> AgentScanT;
-
-    // Shared memory for AgentScan
-    __shared__ typename AgentScanT::TempStorage temp_storage;
-
-    // Process tiles
-    AgentScanT(temp_storage, d_in, d_out, scan_op, init_value).ConsumeRange(
-        num_items,
-        tile_state,
-        start_tile);
-}
-
-
-
-
-/******************************************************************************
- * Dispatch
- ******************************************************************************/
-
-
-/**
- * Utility class for dispatching the appropriately-tuned kernels for DeviceScan
- */
-template <
-    typename InputIteratorT,     ///< Random-access input iterator type for reading scan inputs \iterator
-    typename OutputIteratorT,    ///< Random-access output iterator type for writing scan outputs \iterator
-    typename ScanOpT,            ///< Binary scan functor type having member <tt>T operator()(const T &a, const T &b)</tt>
-    typename InitValueT,          ///< The init_value element type for ScanOpT (cub::NullType for inclusive scans)
-    typename OffsetT>            ///< Signed integer type for global offsets
-struct DispatchScan
-{
-    //---------------------------------------------------------------------
-    // Constants and Types
-    //---------------------------------------------------------------------
-
-    enum
-    {
-        INIT_KERNEL_THREADS = 128
-    };
-
-    // The output value type
-    typedef typename If<(Equals<typename std::iterator_traits<OutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
-        typename std::iterator_traits<InputIteratorT>::value_type,                                          // ... then the input iterator's value type,
-        typename std::iterator_traits<OutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
-
-    // Tile status descriptor interface type
-    typedef ScanTileState<OutputT> ScanTileStateT;
-
-
-    //---------------------------------------------------------------------
-    // Tuning policies
-    //---------------------------------------------------------------------
-
-    /// SM600
-    struct Policy600
-    {
-        typedef AgentScanPolicy<
-            CUB_SCALED_GRANULARITIES(128, 15, OutputT),      ///< Threads per block, items per thread
-                BLOCK_LOAD_TRANSPOSE,
-                LOAD_DEFAULT,
-                BLOCK_STORE_TRANSPOSE,
-                BLOCK_SCAN_WARP_SCANS>
-            ScanPolicyT;
-    };
-
-
-    /// SM520
-    struct Policy520
-    {
-        // Titan X: 32.47B items/s @ 48M 32-bit T
-        typedef AgentScanPolicy<
-                CUB_SCALED_GRANULARITIES(128, 12, OutputT),      ///< Threads per block, items per thread
-                BLOCK_LOAD_DIRECT,
-                LOAD_LDG,
-                BLOCK_STORE_WARP_TRANSPOSE,
-                BLOCK_SCAN_WARP_SCANS>
-            ScanPolicyT;
-    };
-
-
-    /// SM35
-    struct Policy350
-    {
-        // GTX Titan: 29.5B items/s (232.4 GB/s) @ 48M 32-bit T
-        typedef AgentScanPolicy<
-                CUB_SCALED_GRANULARITIES(128, 12, OutputT),      ///< Threads per block, items per thread
-                BLOCK_LOAD_DIRECT,
-                LOAD_LDG,
-                BLOCK_STORE_WARP_TRANSPOSE_TIMESLICED,
-                BLOCK_SCAN_RAKING>
-            ScanPolicyT;
-    };
-
-    /// SM30
-    struct Policy300
-    {
-        typedef AgentScanPolicy<
-                CUB_SCALED_GRANULARITIES(256, 9, OutputT),      ///< Threads per block, items per thread
-                BLOCK_LOAD_WARP_TRANSPOSE,
-                LOAD_DEFAULT,
-                BLOCK_STORE_WARP_TRANSPOSE,
-                BLOCK_SCAN_WARP_SCANS>
-            ScanPolicyT;
-    };
-
-    /// SM20
-    struct Policy200
-    {
-        // GTX 580: 20.3B items/s (162.3 GB/s) @ 48M 32-bit T
-        typedef AgentScanPolicy<
-                CUB_SCALED_GRANULARITIES(128, 12, OutputT),      ///< Threads per block, items per thread
-                BLOCK_LOAD_WARP_TRANSPOSE,
-                LOAD_DEFAULT,
-                BLOCK_STORE_WARP_TRANSPOSE,
-                BLOCK_SCAN_WARP_SCANS>
-            ScanPolicyT;
-    };
-
-    /// SM13
-    struct Policy130
-    {
-        typedef AgentScanPolicy<
-                CUB_SCALED_GRANULARITIES(96, 21, OutputT),      ///< Threads per block, items per thread
-                BLOCK_LOAD_WARP_TRANSPOSE,
-                LOAD_DEFAULT,
-                BLOCK_STORE_WARP_TRANSPOSE,
-                BLOCK_SCAN_RAKING_MEMOIZE>
-            ScanPolicyT;
-    };
-
-    /// SM10
-    struct Policy100
-    {
-        typedef AgentScanPolicy<
-                CUB_SCALED_GRANULARITIES(64, 9, OutputT),      ///< Threads per block, items per thread
-                BLOCK_LOAD_WARP_TRANSPOSE,
-                LOAD_DEFAULT,
-                BLOCK_STORE_WARP_TRANSPOSE,
-                BLOCK_SCAN_WARP_SCANS>
-            ScanPolicyT;
-    };
-
-
-    //---------------------------------------------------------------------
-    // Tuning policies of current PTX compiler pass
-    //---------------------------------------------------------------------
-
-#if (CUB_PTX_ARCH >= 600)
-    typedef Policy600 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 520)
-    typedef Policy520 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 350)
-    typedef Policy350 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 300)
-    typedef Policy300 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 200)
-    typedef Policy200 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 130)
-    typedef Policy130 PtxPolicy;
-
-#else
-    typedef Policy100 PtxPolicy;
-
-#endif
-
-    // "Opaque" policies (whose parameterizations aren't reflected in the type signature)
-    struct PtxAgentScanPolicy : PtxPolicy::ScanPolicyT {};
-
-
-    //---------------------------------------------------------------------
-    // Utilities
-    //---------------------------------------------------------------------
-
-    /**
-     * Initialize kernel dispatch configurations with the policies corresponding to the PTX assembly we will use
-     */
-    template <typename KernelConfig>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static void InitConfigs(
-        int             ptx_version,
-        KernelConfig    &scan_kernel_config)
-    {
-    #if (CUB_PTX_ARCH > 0)
-        (void)ptx_version;
-
-        // We're on the device, so initialize the kernel dispatch configurations with the current PTX policy
-        scan_kernel_config.template Init<PtxAgentScanPolicy>();
-
-    #else
-
-        // We're on the host, so lookup and initialize the kernel dispatch configurations with the policies that match the device's PTX version
-        if (ptx_version >= 600)
-        {
-            scan_kernel_config.template Init<typename Policy600::ScanPolicyT>();
-        }
-        else if (ptx_version >= 520)
-        {
-            scan_kernel_config.template Init<typename Policy520::ScanPolicyT>();
-        }
-        else if (ptx_version >= 350)
-        {
-            scan_kernel_config.template Init<typename Policy350::ScanPolicyT>();
-        }
-        else if (ptx_version >= 300)
-        {
-            scan_kernel_config.template Init<typename Policy300::ScanPolicyT>();
-        }
-        else if (ptx_version >= 200)
-        {
-            scan_kernel_config.template Init<typename Policy200::ScanPolicyT>();
-        }
-        else if (ptx_version >= 130)
-        {
-            scan_kernel_config.template Init<typename Policy130::ScanPolicyT>();
-        }
-        else
-        {
-            scan_kernel_config.template Init<typename Policy100::ScanPolicyT>();
-        }
-
-    #endif
-    }
-
-
-    /**
-     * Kernel kernel dispatch configuration.
-     */
-    struct KernelConfig
-    {
-        int block_threads;
-        int items_per_thread;
-        int tile_items;
-
-        template <typename PolicyT>
-        CUB_RUNTIME_FUNCTION __forceinline__
-        void Init()
-        {
-            block_threads       = PolicyT::BLOCK_THREADS;
-            items_per_thread    = PolicyT::ITEMS_PER_THREAD;
-            tile_items          = block_threads * items_per_thread;
-        }
-    };
-
-
-    //---------------------------------------------------------------------
-    // Dispatch entrypoints
-    //---------------------------------------------------------------------
-
-    /**
-     * Internal dispatch routine for computing a device-wide prefix scan using the
-     * specified kernel functions.
-     */
-    template <
-        typename            ScanInitKernelPtrT,     ///< Function type of cub::DeviceScanInitKernel
-        typename            ScanSweepKernelPtrT>    ///< Function type of cub::DeviceScanKernelPtrT
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Dispatch(
-        void*               d_temp_storage,         ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&             temp_storage_bytes,     ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT      d_in,                   ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT     d_out,                  ///< [out] Pointer to the output sequence of data items
-        ScanOpT             scan_op,                ///< [in] Binary scan functor 
-        InitValueT          init_value,             ///< [in] Initial value to seed the exclusive scan
-        OffsetT             num_items,              ///< [in] Total number of input items (i.e., the length of \p d_in)
-        cudaStream_t        stream,                 ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                debug_synchronous,      ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-        int                 /*ptx_version*/,        ///< [in] PTX version of dispatch kernels
-        ScanInitKernelPtrT  init_kernel,            ///< [in] Kernel function pointer to parameterization of cub::DeviceScanInitKernel
-        ScanSweepKernelPtrT scan_kernel,            ///< [in] Kernel function pointer to parameterization of cub::DeviceScanKernel
-        KernelConfig        scan_kernel_config)     ///< [in] Dispatch parameters that match the policy that \p scan_kernel was compiled for
-    {
-
-#ifndef CUB_RUNTIME_ENABLED
-        (void)d_temp_storage;
-        (void)temp_storage_bytes;
-        (void)d_in;
-        (void)d_out;
-        (void)scan_op;
-        (void)init_value;
-        (void)num_items;
-        (void)stream;
-        (void)debug_synchronous;
-        (void)init_kernel;
-        (void)scan_kernel;
-        (void)scan_kernel_config;
-
-        // Kernel launch not supported from this device
-        return CubDebug(cudaErrorNotSupported);
-
-#else
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Get device ordinal
-            int device_ordinal;
-            if (CubDebug(error = cudaGetDevice(&device_ordinal))) break;
-
-            // Get SM count
-            int sm_count;
-            if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break;
-
-            // Number of input tiles
-            int tile_size = scan_kernel_config.block_threads * scan_kernel_config.items_per_thread;
-            int num_tiles = (num_items + tile_size - 1) / tile_size;
-
-            // Specify temporary storage allocation requirements
-            size_t  allocation_sizes[1];
-            if (CubDebug(error = ScanTileStateT::AllocationSize(num_tiles, allocation_sizes[0]))) break;    // bytes needed for tile status descriptors
-
-            // Compute allocation pointers into the single storage blob (or compute the necessary size of the blob)
-            void* allocations[1];
-            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
-            if (d_temp_storage == NULL)
-            {
-                // Return if the caller is simply requesting the size of the storage allocation
-                break;
-            }
-
-            // Return if empty problem
-            if (num_items == 0)
-                break;
-
-            // Construct the tile status interface
-            ScanTileStateT tile_state;
-            if (CubDebug(error = tile_state.Init(num_tiles, allocations[0], allocation_sizes[0]))) break;
-
-            // Log init_kernel configuration
-            int init_grid_size = (num_tiles + INIT_KERNEL_THREADS - 1) / INIT_KERNEL_THREADS;
-            if (debug_synchronous) _CubLog("Invoking init_kernel<<<%d, %d, 0, %lld>>>()\n", init_grid_size, INIT_KERNEL_THREADS, (long long) stream);
-
-            // Invoke init_kernel to initialize tile descriptors
-            init_kernel<<<init_grid_size, INIT_KERNEL_THREADS, 0, stream>>>(
-                tile_state,
-                num_tiles);
-
-            // Check for failure to launch
-            if (CubDebug(error = cudaPeekAtLastError())) break;
-
-            // Sync the stream if specified to flush runtime errors
-            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-
-            // Get SM occupancy for scan_kernel
-            int scan_sm_occupancy;
-            if (CubDebug(error = MaxSmOccupancy(
-                scan_sm_occupancy,            // out
-                scan_kernel,
-                scan_kernel_config.block_threads))) break;
-
-            // Get max x-dimension of grid
-            int max_dim_x;
-            if (CubDebug(error = cudaDeviceGetAttribute(&max_dim_x, cudaDevAttrMaxGridDimX, device_ordinal))) break;;
-
-            // Run grids in epochs (in case number of tiles exceeds max x-dimension
-            int scan_grid_size = CUB_MIN(num_tiles, max_dim_x);
-            for (int start_tile = 0; start_tile < num_tiles; start_tile += scan_grid_size)
-            {
-                // Log scan_kernel configuration
-                if (debug_synchronous) _CubLog("Invoking %d scan_kernel<<<%d, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
-                    start_tile, scan_grid_size, scan_kernel_config.block_threads, (long long) stream, scan_kernel_config.items_per_thread, scan_sm_occupancy);
-
-                // Invoke scan_kernel
-                scan_kernel<<<scan_grid_size, scan_kernel_config.block_threads, 0, stream>>>(
-                    d_in,
-                    d_out,
-                    tile_state,
-                    start_tile,
-                    scan_op,
-                    init_value,
-                    num_items);
-
-                // Check for failure to launch
-                if (CubDebug(error = cudaPeekAtLastError())) break;
-
-                // Sync the stream if specified to flush runtime errors
-                if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-            }
-        }
-        while (0);
-
-        return error;
-
-#endif  // CUB_RUNTIME_ENABLED
-    }
-
-
-    /**
-     * Internal dispatch routine
-     */
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Dispatch(
-        void*           d_temp_storage,         ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&         temp_storage_bytes,     ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT  d_in,                   ///< [in] Pointer to the input sequence of data items
-        OutputIteratorT d_out,                  ///< [out] Pointer to the output sequence of data items
-        ScanOpT         scan_op,                ///< [in] Binary scan functor 
-        InitValueT      init_value,             ///< [in] Initial value to seed the exclusive scan
-        OffsetT         num_items,              ///< [in] Total number of input items (i.e., the length of \p d_in)
-        cudaStream_t    stream,                 ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool            debug_synchronous)      ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Get PTX version
-            int ptx_version;
-            if (CubDebug(error = PtxVersion(ptx_version))) break;
-
-            // Get kernel kernel dispatch configurations
-            KernelConfig scan_kernel_config;
-            InitConfigs(ptx_version, scan_kernel_config);
-
-            // Dispatch
-            if (CubDebug(error = Dispatch(
-                d_temp_storage,
-                temp_storage_bytes,
-                d_in,
-                d_out,
-                scan_op,
-                init_value,
-                num_items,
-                stream,
-                debug_synchronous,
-                ptx_version,
-                DeviceScanInitKernel<ScanTileStateT>,
-                DeviceScanKernel<PtxAgentScanPolicy, InputIteratorT, OutputIteratorT, ScanTileStateT, ScanOpT, InitValueT, OffsetT>,
-                scan_kernel_config))) break;
-        }
-        while (0);
-
-        return error;
-    }
-};
-
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/device/dispatch/dispatch_select_if.cuh b/pfsimulator/third_party/cub/device/dispatch/dispatch_select_if.cuh
deleted file mode 100644
index 60b331338..000000000
--- a/pfsimulator/third_party/cub/device/dispatch/dispatch_select_if.cuh
+++ /dev/null
@@ -1,542 +0,0 @@
-
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::DeviceSelect provides device-wide, parallel operations for selecting items from sequences of data items residing within device-accessible memory.
- */
-
-#pragma once
-
-#include <stdio.h>
-#include <iterator>
-
-#include "dispatch_scan.cuh"
-#include "../../agent/agent_select_if.cuh"
-#include "../../thread/thread_operators.cuh"
-#include "../../grid/grid_queue.cuh"
-#include "../../util_device.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/******************************************************************************
- * Kernel entry points
- *****************************************************************************/
-
-/**
- * Select kernel entry point (multi-block)
- *
- * Performs functor-based selection if SelectOpT functor type != NullType
- * Otherwise performs flag-based selection if FlagsInputIterator's value type != NullType
- * Otherwise performs discontinuity selection (keep unique)
- */
-template <
-    typename            AgentSelectIfPolicyT,       ///< Parameterized AgentSelectIfPolicyT tuning policy type
-    typename            InputIteratorT,             ///< Random-access input iterator type for reading input items
-    typename            FlagsInputIteratorT,        ///< Random-access input iterator type for reading selection flags (NullType* if a selection functor or discontinuity flagging is to be used for selection)
-    typename            SelectedOutputIteratorT,    ///< Random-access output iterator type for writing selected items
-    typename            NumSelectedIteratorT,       ///< Output iterator type for recording the number of items selected
-    typename            ScanTileStateT,             ///< Tile status interface type
-    typename            SelectOpT,                  ///< Selection operator type (NullType if selection flags or discontinuity flagging is to be used for selection)
-    typename            EqualityOpT,                ///< Equality operator type (NullType if selection functor or selection flags is to be used for selection)
-    typename            OffsetT,                    ///< Signed integer type for global offsets
-    bool                KEEP_REJECTS>               ///< Whether or not we push rejected items to the back of the output
-__launch_bounds__ (int(AgentSelectIfPolicyT::BLOCK_THREADS))
-__global__ void DeviceSelectSweepKernel(
-    InputIteratorT          d_in,                   ///< [in] Pointer to the input sequence of data items
-    FlagsInputIteratorT     d_flags,                ///< [in] Pointer to the input sequence of selection flags (if applicable)
-    SelectedOutputIteratorT d_selected_out,         ///< [out] Pointer to the output sequence of selected data items
-    NumSelectedIteratorT    d_num_selected_out,     ///< [out] Pointer to the total number of items selected (i.e., length of \p d_selected_out)
-    ScanTileStateT          tile_status,            ///< [in] Tile status interface
-    SelectOpT               select_op,              ///< [in] Selection operator
-    EqualityOpT             equality_op,            ///< [in] Equality operator
-    OffsetT                 num_items,              ///< [in] Total number of input items (i.e., length of \p d_in)
-    int                     num_tiles)              ///< [in] Total number of tiles for the entire problem
-{
-    // Thread block type for selecting data from input tiles
-    typedef AgentSelectIf<
-        AgentSelectIfPolicyT,
-        InputIteratorT,
-        FlagsInputIteratorT,
-        SelectedOutputIteratorT,
-        SelectOpT,
-        EqualityOpT,
-        OffsetT,
-        KEEP_REJECTS> AgentSelectIfT;
-
-    // Shared memory for AgentSelectIf
-    __shared__ typename AgentSelectIfT::TempStorage temp_storage;
-
-    // Process tiles
-    AgentSelectIfT(temp_storage, d_in, d_flags, d_selected_out, select_op, equality_op, num_items).ConsumeRange(
-        num_tiles,
-        tile_status,
-        d_num_selected_out);
-}
-
-
-
-
-/******************************************************************************
- * Dispatch
- ******************************************************************************/
-
-/**
- * Utility class for dispatching the appropriately-tuned kernels for DeviceSelect
- */
-template <
-    typename    InputIteratorT,                 ///< Random-access input iterator type for reading input items
-    typename    FlagsInputIteratorT,            ///< Random-access input iterator type for reading selection flags (NullType* if a selection functor or discontinuity flagging is to be used for selection)
-    typename    SelectedOutputIteratorT,        ///< Random-access output iterator type for writing selected items
-    typename    NumSelectedIteratorT,           ///< Output iterator type for recording the number of items selected
-    typename    SelectOpT,                      ///< Selection operator type (NullType if selection flags or discontinuity flagging is to be used for selection)
-    typename    EqualityOpT,                    ///< Equality operator type (NullType if selection functor or selection flags is to be used for selection)
-    typename    OffsetT,                        ///< Signed integer type for global offsets
-    bool        KEEP_REJECTS>                   ///< Whether or not we push rejected items to the back of the output
-struct DispatchSelectIf
-{
-    /******************************************************************************
-     * Types and constants
-     ******************************************************************************/
-
-    // The output value type
-    typedef typename If<(Equals<typename std::iterator_traits<SelectedOutputIteratorT>::value_type, void>::VALUE),  // OutputT =  (if output iterator's value type is void) ?
-        typename std::iterator_traits<InputIteratorT>::value_type,                                                  // ... then the input iterator's value type,
-        typename std::iterator_traits<SelectedOutputIteratorT>::value_type>::Type OutputT;                          // ... else the output iterator's value type
-
-    // The flag value type
-    typedef typename std::iterator_traits<FlagsInputIteratorT>::value_type FlagT;
-
-    enum
-    {
-        INIT_KERNEL_THREADS = 128,
-    };
-
-    // Tile status descriptor interface type
-    typedef ScanTileState<OffsetT> ScanTileStateT;
-
-
-    /******************************************************************************
-     * Tuning policies
-     ******************************************************************************/
-
-    /// SM35
-    struct Policy350
-    {
-        enum {
-            NOMINAL_4B_ITEMS_PER_THREAD = 10,
-            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(OutputT)))),
-        };
-
-        typedef AgentSelectIfPolicy<
-                128,
-                ITEMS_PER_THREAD,
-                BLOCK_LOAD_DIRECT,
-                LOAD_LDG,
-                BLOCK_SCAN_WARP_SCANS>
-            SelectIfPolicyT;
-    };
-
-    /// SM30
-    struct Policy300
-    {
-        enum {
-            NOMINAL_4B_ITEMS_PER_THREAD = 7,
-            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(3, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(OutputT)))),
-        };
-
-        typedef AgentSelectIfPolicy<
-                128,
-                ITEMS_PER_THREAD,
-                BLOCK_LOAD_WARP_TRANSPOSE,
-                LOAD_DEFAULT,
-                BLOCK_SCAN_WARP_SCANS>
-            SelectIfPolicyT;
-    };
-
-    /// SM20
-    struct Policy200
-    {
-        enum {
-            NOMINAL_4B_ITEMS_PER_THREAD = (KEEP_REJECTS) ? 7 : 15,
-            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(OutputT)))),
-        };
-
-        typedef AgentSelectIfPolicy<
-                128,
-                ITEMS_PER_THREAD,
-                BLOCK_LOAD_WARP_TRANSPOSE,
-                LOAD_DEFAULT,
-                BLOCK_SCAN_WARP_SCANS>
-            SelectIfPolicyT;
-    };
-
-    /// SM13
-    struct Policy130
-    {
-        enum {
-            NOMINAL_4B_ITEMS_PER_THREAD = 9,
-            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(OutputT)))),
-        };
-
-        typedef AgentSelectIfPolicy<
-                64,
-                ITEMS_PER_THREAD,
-                BLOCK_LOAD_WARP_TRANSPOSE,
-                LOAD_DEFAULT,
-                BLOCK_SCAN_RAKING_MEMOIZE>
-            SelectIfPolicyT;
-    };
-
-    /// SM10
-    struct Policy100
-    {
-        enum {
-            NOMINAL_4B_ITEMS_PER_THREAD = 9,
-            ITEMS_PER_THREAD            = CUB_MIN(NOMINAL_4B_ITEMS_PER_THREAD, CUB_MAX(1, (NOMINAL_4B_ITEMS_PER_THREAD * 4 / sizeof(OutputT)))),
-        };
-
-        typedef AgentSelectIfPolicy<
-                64,
-                ITEMS_PER_THREAD,
-                BLOCK_LOAD_WARP_TRANSPOSE,
-                LOAD_DEFAULT,
-                BLOCK_SCAN_RAKING>
-            SelectIfPolicyT;
-    };
-
-
-    /******************************************************************************
-     * Tuning policies of current PTX compiler pass
-     ******************************************************************************/
-
-#if (CUB_PTX_ARCH >= 350)
-    typedef Policy350 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 300)
-    typedef Policy300 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 200)
-    typedef Policy200 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 130)
-    typedef Policy130 PtxPolicy;
-
-#else
-    typedef Policy100 PtxPolicy;
-
-#endif
-
-    // "Opaque" policies (whose parameterizations aren't reflected in the type signature)
-    struct PtxSelectIfPolicyT : PtxPolicy::SelectIfPolicyT {};
-
-
-    /******************************************************************************
-     * Utilities
-     ******************************************************************************/
-
-    /**
-     * Initialize kernel dispatch configurations with the policies corresponding to the PTX assembly we will use
-     */
-    template <typename KernelConfig>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static void InitConfigs(
-        int             ptx_version,
-        KernelConfig    &select_if_config)
-    {
-    #if (CUB_PTX_ARCH > 0)
-        (void)ptx_version;
-
-        // We're on the device, so initialize the kernel dispatch configurations with the current PTX policy
-        select_if_config.template Init<PtxSelectIfPolicyT>();
-
-    #else
-
-        // We're on the host, so lookup and initialize the kernel dispatch configurations with the policies that match the device's PTX version
-        if (ptx_version >= 350)
-        {
-            select_if_config.template Init<typename Policy350::SelectIfPolicyT>();
-        }
-        else if (ptx_version >= 300)
-        {
-            select_if_config.template Init<typename Policy300::SelectIfPolicyT>();
-        }
-        else if (ptx_version >= 200)
-        {
-            select_if_config.template Init<typename Policy200::SelectIfPolicyT>();
-        }
-        else if (ptx_version >= 130)
-        {
-            select_if_config.template Init<typename Policy130::SelectIfPolicyT>();
-        }
-        else
-        {
-            select_if_config.template Init<typename Policy100::SelectIfPolicyT>();
-        }
-
-    #endif
-    }
-
-
-    /**
-     * Kernel kernel dispatch configuration.
-     */
-    struct KernelConfig
-    {
-        int block_threads;
-        int items_per_thread;
-        int tile_items;
-
-        template <typename PolicyT>
-        CUB_RUNTIME_FUNCTION __forceinline__
-        void Init()
-        {
-            block_threads       = PolicyT::BLOCK_THREADS;
-            items_per_thread    = PolicyT::ITEMS_PER_THREAD;
-            tile_items          = block_threads * items_per_thread;
-        }
-    };
-
-
-    /******************************************************************************
-     * Dispatch entrypoints
-     ******************************************************************************/
-
-    /**
-     * Internal dispatch routine for computing a device-wide selection using the
-     * specified kernel functions.
-     */
-    template <
-        typename                    ScanInitKernelPtrT,             ///< Function type of cub::DeviceScanInitKernel
-        typename                    SelectIfKernelPtrT>             ///< Function type of cub::SelectIfKernelPtrT
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Dispatch(
-        void*                       d_temp_storage,                 ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&                     temp_storage_bytes,             ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT              d_in,                           ///< [in] Pointer to the input sequence of data items
-        FlagsInputIteratorT         d_flags,                        ///< [in] Pointer to the input sequence of selection flags (if applicable)
-        SelectedOutputIteratorT     d_selected_out,                 ///< [in] Pointer to the output sequence of selected data items
-        NumSelectedIteratorT        d_num_selected_out,             ///< [in] Pointer to the total number of items selected (i.e., length of \p d_selected_out)
-        SelectOpT                   select_op,                      ///< [in] Selection operator
-        EqualityOpT                 equality_op,                    ///< [in] Equality operator
-        OffsetT                     num_items,                      ///< [in] Total number of input items (i.e., length of \p d_in)
-        cudaStream_t                stream,                         ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous,              ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-        int                         /*ptx_version*/,                ///< [in] PTX version of dispatch kernels
-        ScanInitKernelPtrT          scan_init_kernel,               ///< [in] Kernel function pointer to parameterization of cub::DeviceScanInitKernel
-        SelectIfKernelPtrT          select_if_kernel,               ///< [in] Kernel function pointer to parameterization of cub::DeviceSelectSweepKernel
-        KernelConfig                select_if_config)               ///< [in] Dispatch parameters that match the policy that \p select_if_kernel was compiled for
-    {
-
-#ifndef CUB_RUNTIME_ENABLED
-        (void)d_temp_storage;
-        (void)temp_storage_bytes;
-        (void)d_in;
-        (void)d_flags;
-        (void)d_selected_out;
-        (void)d_num_selected_out;
-        (void)select_op;
-        (void)equality_op;
-        (void)num_items;
-        (void)stream;
-        (void)debug_synchronous;
-        (void)scan_init_kernel;
-        (void)select_if_kernel;
-        (void)select_if_config;
-
-        // Kernel launch not supported from this device
-        return CubDebug(cudaErrorNotSupported);
-
-#else
-
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Get device ordinal
-            int device_ordinal;
-            if (CubDebug(error = cudaGetDevice(&device_ordinal))) break;
-
-            // Get SM count
-            int sm_count;
-            if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break;
-
-            // Number of input tiles
-            int tile_size = select_if_config.block_threads * select_if_config.items_per_thread;
-            int num_tiles = (num_items + tile_size - 1) / tile_size;
-
-            // Specify temporary storage allocation requirements
-            size_t  allocation_sizes[1];
-            if (CubDebug(error = ScanTileStateT::AllocationSize(num_tiles, allocation_sizes[0]))) break;    // bytes needed for tile status descriptors
-
-            // Compute allocation pointers into the single storage blob (or compute the necessary size of the blob)
-            void* allocations[1];
-            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
-            if (d_temp_storage == NULL)
-            {
-                // Return if the caller is simply requesting the size of the storage allocation
-                break;
-            }
-
-            // Construct the tile status interface
-            ScanTileStateT tile_status;
-            if (CubDebug(error = tile_status.Init(num_tiles, allocations[0], allocation_sizes[0]))) break;
-
-            // Log scan_init_kernel configuration
-            int init_grid_size = CUB_MAX(1, (num_tiles + INIT_KERNEL_THREADS - 1) / INIT_KERNEL_THREADS);
-            if (debug_synchronous) _CubLog("Invoking scan_init_kernel<<<%d, %d, 0, %lld>>>()\n", init_grid_size, INIT_KERNEL_THREADS, (long long) stream);
-
-            // Invoke scan_init_kernel to initialize tile descriptors
-            scan_init_kernel<<<init_grid_size, INIT_KERNEL_THREADS, 0, stream>>>(
-                tile_status,
-                num_tiles,
-                d_num_selected_out);
-
-            // Check for failure to launch
-            if (CubDebug(error = cudaPeekAtLastError())) break;
-
-            // Sync the stream if specified to flush runtime errors
-            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-
-            // Return if empty problem
-            if (num_items == 0)
-                break;
-
-            // Get SM occupancy for select_if_kernel
-            int range_select_sm_occupancy;
-            if (CubDebug(error = MaxSmOccupancy(
-                range_select_sm_occupancy,            // out
-                select_if_kernel,
-                select_if_config.block_threads))) break;
-
-            // Get max x-dimension of grid
-            int max_dim_x;
-            if (CubDebug(error = cudaDeviceGetAttribute(&max_dim_x, cudaDevAttrMaxGridDimX, device_ordinal))) break;;
-
-            // Get grid size for scanning tiles
-            dim3 scan_grid_size;
-            scan_grid_size.z = 1;
-            scan_grid_size.y = ((unsigned int) num_tiles + max_dim_x - 1) / max_dim_x;
-            scan_grid_size.x = CUB_MIN(num_tiles, max_dim_x);
-
-            // Log select_if_kernel configuration
-            if (debug_synchronous) _CubLog("Invoking select_if_kernel<<<{%d,%d,%d}, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
-                scan_grid_size.x, scan_grid_size.y, scan_grid_size.z, select_if_config.block_threads, (long long) stream, select_if_config.items_per_thread, range_select_sm_occupancy);
-
-            // Invoke select_if_kernel
-            select_if_kernel<<<scan_grid_size, select_if_config.block_threads, 0, stream>>>(
-                d_in,
-                d_flags,
-                d_selected_out,
-                d_num_selected_out,
-                tile_status,
-                select_op,
-                equality_op,
-                num_items,
-                num_tiles);
-
-            // Check for failure to launch
-            if (CubDebug(error = cudaPeekAtLastError())) break;
-
-            // Sync the stream if specified to flush runtime errors
-            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-        }
-        while (0);
-
-        return error;
-
-#endif  // CUB_RUNTIME_ENABLED
-    }
-
-
-    /**
-     * Internal dispatch routine
-     */
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Dispatch(
-        void*                       d_temp_storage,                 ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&                     temp_storage_bytes,             ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        InputIteratorT              d_in,                           ///< [in] Pointer to the input sequence of data items
-        FlagsInputIteratorT         d_flags,                        ///< [in] Pointer to the input sequence of selection flags (if applicable)
-        SelectedOutputIteratorT     d_selected_out,                 ///< [in] Pointer to the output sequence of selected data items
-        NumSelectedIteratorT        d_num_selected_out,             ///< [in] Pointer to the total number of items selected (i.e., length of \p d_selected_out)
-        SelectOpT                   select_op,                      ///< [in] Selection operator
-        EqualityOpT                 equality_op,                    ///< [in] Equality operator
-        OffsetT                     num_items,                      ///< [in] Total number of input items (i.e., length of \p d_in)
-        cudaStream_t                stream,                         ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                        debug_synchronous)              ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-    {
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Get PTX version
-            int ptx_version;
-    #if (CUB_PTX_ARCH == 0)
-            if (CubDebug(error = PtxVersion(ptx_version))) break;
-    #else
-            ptx_version = CUB_PTX_ARCH;
-    #endif
-
-            // Get kernel kernel dispatch configurations
-            KernelConfig select_if_config;
-            InitConfigs(ptx_version, select_if_config);
-
-            // Dispatch
-            if (CubDebug(error = Dispatch(
-                d_temp_storage,
-                temp_storage_bytes,
-                d_in,
-                d_flags,
-                d_selected_out,
-                d_num_selected_out,
-                select_op,
-                equality_op,
-                num_items,
-                stream,
-                debug_synchronous,
-                ptx_version,
-                DeviceCompactInitKernel<ScanTileStateT, NumSelectedIteratorT>,
-                DeviceSelectSweepKernel<PtxSelectIfPolicyT, InputIteratorT, FlagsInputIteratorT, SelectedOutputIteratorT, NumSelectedIteratorT, ScanTileStateT, SelectOpT, EqualityOpT, OffsetT, KEEP_REJECTS>,
-                select_if_config))) break;
-        }
-        while (0);
-
-        return error;
-    }
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/device/dispatch/dispatch_spmv_orig.cuh b/pfsimulator/third_party/cub/device/dispatch/dispatch_spmv_orig.cuh
deleted file mode 100644
index ab9c5346d..000000000
--- a/pfsimulator/third_party/cub/device/dispatch/dispatch_spmv_orig.cuh
+++ /dev/null
@@ -1,834 +0,0 @@
-
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::DeviceSpmv provides device-wide parallel operations for performing sparse-matrix * vector multiplication (SpMV).
- */
-
-#pragma once
-
-#include <stdio.h>
-#include <iterator>
-
-#include "../../agent/single_pass_scan_operators.cuh"
-#include "../../agent/agent_segment_fixup.cuh"
-#include "../../agent/agent_spmv_orig.cuh"
-#include "../../util_type.cuh"
-#include "../../util_debug.cuh"
-#include "../../util_device.cuh"
-#include "../../thread/thread_search.cuh"
-#include "../../grid/grid_queue.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/******************************************************************************
- * SpMV kernel entry points
- *****************************************************************************/
-
-/**
- * Spmv search kernel. Identifies merge path starting coordinates for each tile.
- */
-template <
-    typename    AgentSpmvPolicyT,           ///< Parameterized SpmvPolicy tuning policy type
-    typename    ValueT,                     ///< Matrix and vector value type
-    typename    OffsetT>                    ///< Signed integer type for sequence offsets
-__global__ void DeviceSpmv1ColKernel(
-    SpmvParams<ValueT, OffsetT> spmv_params)                ///< [in] SpMV input parameter bundle
-{
-    typedef CacheModifiedInputIterator<
-            AgentSpmvPolicyT::VECTOR_VALUES_LOAD_MODIFIER,
-            ValueT,
-            OffsetT>
-        VectorValueIteratorT;
-
-    VectorValueIteratorT wrapped_vector_x(spmv_params.d_vector_x);
-
-    int row_idx = (blockIdx.x * blockDim.x) + threadIdx.x;
-    if (row_idx < spmv_params.num_rows)
-    {
-        OffsetT     end_nonzero_idx = spmv_params.d_row_end_offsets[row_idx];
-        OffsetT     nonzero_idx = spmv_params.d_row_end_offsets[row_idx - 1];
-
-        ValueT value = 0.0;
-        if (end_nonzero_idx != nonzero_idx)
-        {
-            value = spmv_params.d_values[nonzero_idx] * wrapped_vector_x[spmv_params.d_column_indices[nonzero_idx]];
-        }
-
-        spmv_params.d_vector_y[row_idx] = value;
-    }
-}
-
-
-/**
- * Spmv search kernel. Identifies merge path starting coordinates for each tile.
- */
-template <
-    typename    SpmvPolicyT,                    ///< Parameterized SpmvPolicy tuning policy type
-    typename    OffsetT,                        ///< Signed integer type for sequence offsets
-    typename    CoordinateT,                    ///< Merge path coordinate type
-    typename    SpmvParamsT>                    ///< SpmvParams type
-__global__ void DeviceSpmvSearchKernel(
-    int             num_merge_tiles,            ///< [in] Number of SpMV merge tiles (spmv grid size)
-    CoordinateT*    d_tile_coordinates,         ///< [out] Pointer to the temporary array of tile starting coordinates
-    SpmvParamsT     spmv_params)                ///< [in] SpMV input parameter bundle
-{
-    /// Constants
-    enum
-    {
-        BLOCK_THREADS           = SpmvPolicyT::BLOCK_THREADS,
-        ITEMS_PER_THREAD        = SpmvPolicyT::ITEMS_PER_THREAD,
-        TILE_ITEMS              = BLOCK_THREADS * ITEMS_PER_THREAD,
-    };
-
-    typedef CacheModifiedInputIterator<
-            SpmvPolicyT::ROW_OFFSETS_SEARCH_LOAD_MODIFIER,
-            OffsetT,
-            OffsetT>
-        RowOffsetsSearchIteratorT;
-
-    // Find the starting coordinate for all tiles (plus the end coordinate of the last one)
-    int tile_idx = (blockIdx.x * blockDim.x) + threadIdx.x;
-    if (tile_idx < num_merge_tiles + 1)
-    {
-        OffsetT                         diagonal = (tile_idx * TILE_ITEMS);
-        CoordinateT                     tile_coordinate;
-        CountingInputIterator<OffsetT>  nonzero_indices(0);
-
-        // Search the merge path
-        MergePathSearch(
-            diagonal,
-            RowOffsetsSearchIteratorT(spmv_params.d_row_end_offsets),
-            nonzero_indices,
-            spmv_params.num_rows,
-            spmv_params.num_nonzeros,
-            tile_coordinate);
-
-        // Output starting offset
-        d_tile_coordinates[tile_idx] = tile_coordinate;
-    }
-}
-
-
-/**
- * Spmv agent entry point
- */
-template <
-    typename        SpmvPolicyT,                ///< Parameterized SpmvPolicy tuning policy type
-    typename        ScanTileStateT,             ///< Tile status interface type
-    typename        ValueT,                     ///< Matrix and vector value type
-    typename        OffsetT,                    ///< Signed integer type for sequence offsets
-    typename        CoordinateT,                ///< Merge path coordinate type
-    bool            HAS_ALPHA,                  ///< Whether the input parameter Alpha is 1
-    bool            HAS_BETA>                   ///< Whether the input parameter Beta is 0
-__launch_bounds__ (int(SpmvPolicyT::BLOCK_THREADS))
-__global__ void DeviceSpmvKernel(
-    SpmvParams<ValueT, OffsetT>     spmv_params,                ///< [in] SpMV input parameter bundle
-    CoordinateT*                    d_tile_coordinates,         ///< [in] Pointer to the temporary array of tile starting coordinates
-    KeyValuePair<OffsetT,ValueT>*   d_tile_carry_pairs,         ///< [out] Pointer to the temporary array carry-out dot product row-ids, one per block
-    int                             num_tiles,                  ///< [in] Number of merge tiles
-    ScanTileStateT                  tile_state,                 ///< [in] Tile status interface for fixup reduce-by-key kernel
-    int                             num_segment_fixup_tiles)    ///< [in] Number of reduce-by-key tiles (fixup grid size)
-{
-    // Spmv agent type specialization
-    typedef AgentSpmv<
-            SpmvPolicyT,
-            ValueT,
-            OffsetT,
-            HAS_ALPHA,
-            HAS_BETA>
-        AgentSpmvT;
-
-    // Shared memory for AgentSpmv
-    __shared__ typename AgentSpmvT::TempStorage temp_storage;
-
-    AgentSpmvT(temp_storage, spmv_params).ConsumeTile(
-        d_tile_coordinates,
-        d_tile_carry_pairs,
-        num_tiles);
-
-    // Initialize fixup tile status
-    tile_state.InitializeStatus(num_segment_fixup_tiles);
-
-}
-
-
-/**
- * Multi-block reduce-by-key sweep kernel entry point
- */
-template <
-    typename    AgentSegmentFixupPolicyT,       ///< Parameterized AgentSegmentFixupPolicy tuning policy type
-    typename    PairsInputIteratorT,            ///< Random-access input iterator type for keys
-    typename    AggregatesOutputIteratorT,      ///< Random-access output iterator type for values
-    typename    OffsetT,                        ///< Signed integer type for global offsets
-    typename    ScanTileStateT>                 ///< Tile status interface type
-__launch_bounds__ (int(AgentSegmentFixupPolicyT::BLOCK_THREADS))
-__global__ void DeviceSegmentFixupKernel(
-    PairsInputIteratorT         d_pairs_in,         ///< [in] Pointer to the array carry-out dot product row-ids, one per spmv block
-    AggregatesOutputIteratorT   d_aggregates_out,   ///< [in,out] Output value aggregates
-    OffsetT                     num_items,          ///< [in] Total number of items to select from
-    int                         num_tiles,          ///< [in] Total number of tiles for the entire problem
-    ScanTileStateT              tile_state)         ///< [in] Tile status interface
-{
-    // Thread block type for reducing tiles of value segments
-    typedef AgentSegmentFixup<
-            AgentSegmentFixupPolicyT,
-            PairsInputIteratorT,
-            AggregatesOutputIteratorT,
-            cub::Equality,
-            cub::Sum,
-            OffsetT>
-        AgentSegmentFixupT;
-
-    // Shared memory for AgentSegmentFixup
-    __shared__ typename AgentSegmentFixupT::TempStorage temp_storage;
-
-    // Process tiles
-    AgentSegmentFixupT(temp_storage, d_pairs_in, d_aggregates_out, cub::Equality(), cub::Sum()).ConsumeRange(
-        num_items,
-        num_tiles,
-        tile_state);
-}
-
-
-/******************************************************************************
- * Dispatch
- ******************************************************************************/
-
-/**
- * Utility class for dispatching the appropriately-tuned kernels for DeviceSpmv
- */
-template <
-    typename    ValueT,                     ///< Matrix and vector value type
-    typename    OffsetT>                    ///< Signed integer type for global offsets
-struct DispatchSpmv
-{
-    //---------------------------------------------------------------------
-    // Constants and Types
-    //---------------------------------------------------------------------
-
-    enum
-    {
-        INIT_KERNEL_THREADS = 128
-    };
-
-    // SpmvParams bundle type
-    typedef SpmvParams<ValueT, OffsetT> SpmvParamsT;
-
-    // 2D merge path coordinate type
-    typedef typename CubVector<OffsetT, 2>::Type CoordinateT;
-
-    // Tile status descriptor interface type
-    typedef ReduceByKeyScanTileState<ValueT, OffsetT> ScanTileStateT;
-
-    // Tuple type for scanning (pairs accumulated segment-value with segment-index)
-    typedef KeyValuePair<OffsetT, ValueT> KeyValuePairT;
-
-
-    //---------------------------------------------------------------------
-    // Tuning policies
-    //---------------------------------------------------------------------
-
-    /// SM11
-    struct Policy110
-    {
-        typedef AgentSpmvPolicy<
-                128,
-                1,
-                LOAD_DEFAULT,
-                LOAD_DEFAULT,
-                LOAD_DEFAULT,
-                LOAD_DEFAULT,
-                LOAD_DEFAULT,
-                false,
-                BLOCK_SCAN_WARP_SCANS>
-            SpmvPolicyT;
-
-        typedef AgentSegmentFixupPolicy<
-                128,
-                4,
-                BLOCK_LOAD_VECTORIZE,
-                LOAD_DEFAULT,
-                BLOCK_SCAN_WARP_SCANS>
-            SegmentFixupPolicyT;
-    };
-
-    /// SM20
-    struct Policy200 
-    {
-        typedef AgentSpmvPolicy<
-                96,
-                18,
-                LOAD_DEFAULT,
-                LOAD_DEFAULT,
-                LOAD_DEFAULT,
-                LOAD_DEFAULT,
-                LOAD_DEFAULT,
-                false,
-                BLOCK_SCAN_RAKING>
-            SpmvPolicyT;
-
-        typedef AgentSegmentFixupPolicy<
-                128,
-                4,
-                BLOCK_LOAD_VECTORIZE,
-                LOAD_DEFAULT,
-                BLOCK_SCAN_WARP_SCANS>
-            SegmentFixupPolicyT;
-
-    };
-
-
-
-    /// SM30
-    struct Policy300 
-    {
-        typedef AgentSpmvPolicy<
-                96,
-                6,
-                LOAD_DEFAULT,
-                LOAD_DEFAULT,
-                LOAD_DEFAULT,
-                LOAD_DEFAULT,
-                LOAD_DEFAULT,
-                false,
-                BLOCK_SCAN_WARP_SCANS>
-            SpmvPolicyT;
-
-        typedef AgentSegmentFixupPolicy<
-                128,
-                4,
-                BLOCK_LOAD_VECTORIZE,
-                LOAD_DEFAULT,
-                BLOCK_SCAN_WARP_SCANS>
-            SegmentFixupPolicyT;
-
-    };
-
-
-    /// SM35
-    struct Policy350
-    {
-        typedef AgentSpmvPolicy<
-                (sizeof(ValueT) > 4) ? 96 : 128,
-                (sizeof(ValueT) > 4) ? 4 : 7,
-                LOAD_LDG,
-                LOAD_CA,
-                LOAD_LDG,
-                LOAD_LDG,
-                LOAD_LDG,
-                (sizeof(ValueT) > 4) ? true : false,
-                BLOCK_SCAN_WARP_SCANS>
-            SpmvPolicyT;
-
-        typedef AgentSegmentFixupPolicy<
-                128,
-                3,
-                BLOCK_LOAD_VECTORIZE,
-                LOAD_LDG,
-                BLOCK_SCAN_WARP_SCANS>
-            SegmentFixupPolicyT;
-    };
-
-
-    /// SM37
-    struct Policy370
-    {
-
-        typedef AgentSpmvPolicy<
-                (sizeof(ValueT) > 4) ? 128 : 128,
-                (sizeof(ValueT) > 4) ? 9 : 14,
-                LOAD_LDG,
-                LOAD_CA,
-                LOAD_LDG,
-                LOAD_LDG,
-                LOAD_LDG,
-                false, 
-                BLOCK_SCAN_WARP_SCANS>
-            SpmvPolicyT;
-
-        typedef AgentSegmentFixupPolicy<
-                128,
-                3,
-                BLOCK_LOAD_VECTORIZE,
-                LOAD_LDG,
-                BLOCK_SCAN_WARP_SCANS>
-            SegmentFixupPolicyT;
-    };
-
-    /// SM50
-    struct Policy500
-    {
-        typedef AgentSpmvPolicy<
-                (sizeof(ValueT) > 4) ? 64 : 128,
-                (sizeof(ValueT) > 4) ? 6 : 7,
-                LOAD_LDG,
-                LOAD_DEFAULT,
-                (sizeof(ValueT) > 4) ? LOAD_LDG : LOAD_DEFAULT,
-                (sizeof(ValueT) > 4) ? LOAD_LDG : LOAD_DEFAULT,
-                LOAD_LDG,
-                (sizeof(ValueT) > 4) ? true : false,
-                (sizeof(ValueT) > 4) ? BLOCK_SCAN_WARP_SCANS : BLOCK_SCAN_RAKING_MEMOIZE>
-            SpmvPolicyT;
-
-
-        typedef AgentSegmentFixupPolicy<
-                128,
-                3,
-                BLOCK_LOAD_VECTORIZE,
-                LOAD_LDG,
-                BLOCK_SCAN_RAKING_MEMOIZE>
-            SegmentFixupPolicyT;
-    };
-
-
-    /// SM60
-    struct Policy600
-    {
-        typedef AgentSpmvPolicy<
-                (sizeof(ValueT) > 4) ? 64 : 128,
-                (sizeof(ValueT) > 4) ? 5 : 7,
-                LOAD_DEFAULT,
-                LOAD_DEFAULT,
-                LOAD_DEFAULT,
-                LOAD_DEFAULT,
-                LOAD_DEFAULT,
-                false,
-                BLOCK_SCAN_WARP_SCANS>
-            SpmvPolicyT;
-
-
-        typedef AgentSegmentFixupPolicy<
-                128,
-                3,
-                BLOCK_LOAD_DIRECT,
-                LOAD_LDG,
-                BLOCK_SCAN_WARP_SCANS>
-            SegmentFixupPolicyT;
-    };
-
-
-
-    //---------------------------------------------------------------------
-    // Tuning policies of current PTX compiler pass
-    //---------------------------------------------------------------------
-
-#if (CUB_PTX_ARCH >= 600)
-    typedef Policy600 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 500)
-    typedef Policy500 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 370)
-    typedef Policy370 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 350)
-    typedef Policy350 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 300)
-    typedef Policy300 PtxPolicy;
-
-#elif (CUB_PTX_ARCH >= 200)
-    typedef Policy200 PtxPolicy;
-
-#else
-    typedef Policy110 PtxPolicy;
-
-#endif
-
-    // "Opaque" policies (whose parameterizations aren't reflected in the type signature)
-    struct PtxSpmvPolicyT : PtxPolicy::SpmvPolicyT {};
-    struct PtxSegmentFixupPolicy : PtxPolicy::SegmentFixupPolicyT {};
-
-
-    //---------------------------------------------------------------------
-    // Utilities
-    //---------------------------------------------------------------------
-
-    /**
-     * Initialize kernel dispatch configurations with the policies corresponding to the PTX assembly we will use
-     */
-    template <typename KernelConfig>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static void InitConfigs(
-        int             ptx_version,
-        KernelConfig    &spmv_config,
-        KernelConfig    &segment_fixup_config)
-    {
-    #if (CUB_PTX_ARCH > 0)
-
-        // We're on the device, so initialize the kernel dispatch configurations with the current PTX policy
-        spmv_config.template Init<PtxSpmvPolicyT>();
-        segment_fixup_config.template Init<PtxSegmentFixupPolicy>();
-
-    #else
-
-        // We're on the host, so lookup and initialize the kernel dispatch configurations with the policies that match the device's PTX version
-        if (ptx_version >= 600)
-        {
-            spmv_config.template            Init<typename Policy600::SpmvPolicyT>();
-            segment_fixup_config.template   Init<typename Policy600::SegmentFixupPolicyT>();
-        }
-        else if (ptx_version >= 500)
-        {
-            spmv_config.template            Init<typename Policy500::SpmvPolicyT>();
-            segment_fixup_config.template   Init<typename Policy500::SegmentFixupPolicyT>();
-        }
-        else if (ptx_version >= 370)
-        {
-            spmv_config.template            Init<typename Policy370::SpmvPolicyT>();
-            segment_fixup_config.template   Init<typename Policy370::SegmentFixupPolicyT>();
-        }
-        else if (ptx_version >= 350)
-        {
-            spmv_config.template            Init<typename Policy350::SpmvPolicyT>();
-            segment_fixup_config.template   Init<typename Policy350::SegmentFixupPolicyT>();
-        }
-        else if (ptx_version >= 300)
-        {
-            spmv_config.template            Init<typename Policy300::SpmvPolicyT>();
-            segment_fixup_config.template   Init<typename Policy300::SegmentFixupPolicyT>();
-
-        }
-        else if (ptx_version >= 200)
-        {
-            spmv_config.template            Init<typename Policy200::SpmvPolicyT>();
-            segment_fixup_config.template   Init<typename Policy200::SegmentFixupPolicyT>();
-        }
-        else
-        {
-            spmv_config.template            Init<typename Policy110::SpmvPolicyT>();
-            segment_fixup_config.template   Init<typename Policy110::SegmentFixupPolicyT>();
-        }
-
-    #endif
-    }
-
-
-    /**
-     * Kernel kernel dispatch configuration.
-     */
-    struct KernelConfig
-    {
-        int block_threads;
-        int items_per_thread;
-        int tile_items;
-
-        template <typename PolicyT>
-        CUB_RUNTIME_FUNCTION __forceinline__
-        void Init()
-        {
-            block_threads       = PolicyT::BLOCK_THREADS;
-            items_per_thread    = PolicyT::ITEMS_PER_THREAD;
-            tile_items          = block_threads * items_per_thread;
-        }
-    };
-
-
-    //---------------------------------------------------------------------
-    // Dispatch entrypoints
-    //---------------------------------------------------------------------
-
-    /**
-     * Internal dispatch routine for computing a device-wide reduction using the
-     * specified kernel functions.
-     *
-     * If the input is larger than a single tile, this method uses two-passes of
-     * kernel invocations.
-     */
-    template <
-        typename                Spmv1ColKernelT,                    ///< Function type of cub::DeviceSpmv1ColKernel
-        typename                SpmvSearchKernelT,                  ///< Function type of cub::AgentSpmvSearchKernel
-        typename                SpmvKernelT,                        ///< Function type of cub::AgentSpmvKernel
-        typename                SegmentFixupKernelT>                 ///< Function type of cub::DeviceSegmentFixupKernelT
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Dispatch(
-        void*                   d_temp_storage,                     ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&                 temp_storage_bytes,                 ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        SpmvParamsT&            spmv_params,                        ///< SpMV input parameter bundle
-        cudaStream_t            stream,                             ///< [in] CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                    debug_synchronous,                  ///< [in] Whether or not to synchronize the stream after every kernel launch to check for errors.  Also causes launch configurations to be printed to the console.  Default is \p false.
-        Spmv1ColKernelT         spmv_1col_kernel,                   ///< [in] Kernel function pointer to parameterization of DeviceSpmv1ColKernel
-        SpmvSearchKernelT       spmv_search_kernel,                 ///< [in] Kernel function pointer to parameterization of AgentSpmvSearchKernel
-        SpmvKernelT             spmv_kernel,                        ///< [in] Kernel function pointer to parameterization of AgentSpmvKernel
-        SegmentFixupKernelT     segment_fixup_kernel,               ///< [in] Kernel function pointer to parameterization of cub::DeviceSegmentFixupKernel
-        KernelConfig            spmv_config,                        ///< [in] Dispatch parameters that match the policy that \p spmv_kernel was compiled for
-        KernelConfig            segment_fixup_config)               ///< [in] Dispatch parameters that match the policy that \p segment_fixup_kernel was compiled for
-    {
-#ifndef CUB_RUNTIME_ENABLED
-
-        // Kernel launch not supported from this device
-        return CubDebug(cudaErrorNotSupported );
-
-#else
-        cudaError error = cudaSuccess;
-        do
-        {
-            if (spmv_params.num_cols == 1)
-            {
-                if (d_temp_storage == NULL)
-                {
-                    // Return if the caller is simply requesting the size of the storage allocation
-                    temp_storage_bytes = 1;
-                    break;
-                }
-
-                // Get search/init grid dims
-                int degen_col_kernel_block_size     = INIT_KERNEL_THREADS;
-                int degen_col_kernel_grid_size      = (spmv_params.num_rows + degen_col_kernel_block_size - 1) / degen_col_kernel_block_size;
-
-                if (debug_synchronous) _CubLog("Invoking spmv_1col_kernel<<<%d, %d, 0, %lld>>>()\n",
-                    degen_col_kernel_grid_size, degen_col_kernel_block_size, (long long) stream);
-
-                // Invoke spmv_search_kernel
-                spmv_1col_kernel<<<degen_col_kernel_grid_size, degen_col_kernel_block_size, 0, stream>>>(
-                    spmv_params);
-
-                // Check for failure to launch
-                if (CubDebug(error = cudaPeekAtLastError())) break;
-
-                // Sync the stream if specified to flush runtime errors
-                if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-
-                break;
-            }
-
-            // Get device ordinal
-            int device_ordinal;
-            if (CubDebug(error = cudaGetDevice(&device_ordinal))) break;
-
-            // Get SM count
-            int sm_count;
-            if (CubDebug(error = cudaDeviceGetAttribute (&sm_count, cudaDevAttrMultiProcessorCount, device_ordinal))) break;
-
-            // Get max x-dimension of grid
-            int max_dim_x;
-            if (CubDebug(error = cudaDeviceGetAttribute(&max_dim_x, cudaDevAttrMaxGridDimX, device_ordinal))) break;;
-
-            // Total number of spmv work items
-            int num_merge_items = spmv_params.num_rows + spmv_params.num_nonzeros;
-
-            // Tile sizes of kernels
-            int merge_tile_size              = spmv_config.block_threads * spmv_config.items_per_thread;
-            int segment_fixup_tile_size     = segment_fixup_config.block_threads * segment_fixup_config.items_per_thread;
-
-            // Number of tiles for kernels
-            unsigned int num_merge_tiles            = (num_merge_items + merge_tile_size - 1) / merge_tile_size;
-            unsigned int num_segment_fixup_tiles    = (num_merge_tiles + segment_fixup_tile_size - 1) / segment_fixup_tile_size;
-
-            // Get SM occupancy for kernels
-            int spmv_sm_occupancy;
-            if (CubDebug(error = MaxSmOccupancy(
-                spmv_sm_occupancy,
-                spmv_kernel,
-                spmv_config.block_threads))) break;
-
-            int segment_fixup_sm_occupancy;
-            if (CubDebug(error = MaxSmOccupancy(
-                segment_fixup_sm_occupancy,
-                segment_fixup_kernel,
-                segment_fixup_config.block_threads))) break;
-
-            // Get grid dimensions
-            dim3 spmv_grid_size(
-                CUB_MIN(num_merge_tiles, max_dim_x),
-                (num_merge_tiles + max_dim_x - 1) / max_dim_x,
-                1);
-
-            dim3 segment_fixup_grid_size(
-                CUB_MIN(num_segment_fixup_tiles, max_dim_x),
-                (num_segment_fixup_tiles + max_dim_x - 1) / max_dim_x,
-                1);
-
-            // Get the temporary storage allocation requirements
-            size_t allocation_sizes[3];
-            if (CubDebug(error = ScanTileStateT::AllocationSize(num_segment_fixup_tiles, allocation_sizes[0]))) break;    // bytes needed for reduce-by-key tile status descriptors
-            allocation_sizes[1] = num_merge_tiles * sizeof(KeyValuePairT);       // bytes needed for block carry-out pairs
-            allocation_sizes[2] = (num_merge_tiles + 1) * sizeof(CoordinateT);   // bytes needed for tile starting coordinates
-
-            // Alias the temporary allocations from the single storage blob (or compute the necessary size of the blob)
-            void* allocations[3];
-            if (CubDebug(error = AliasTemporaries(d_temp_storage, temp_storage_bytes, allocations, allocation_sizes))) break;
-            if (d_temp_storage == NULL)
-            {
-                // Return if the caller is simply requesting the size of the storage allocation
-                break;
-            }
-
-            // Construct the tile status interface
-            ScanTileStateT tile_state;
-            if (CubDebug(error = tile_state.Init(num_segment_fixup_tiles, allocations[0], allocation_sizes[0]))) break;
-
-            // Alias the other allocations
-            KeyValuePairT*  d_tile_carry_pairs      = (KeyValuePairT*) allocations[1];  // Agent carry-out pairs
-            CoordinateT*    d_tile_coordinates      = (CoordinateT*) allocations[2];    // Agent starting coordinates
-
-            // Get search/init grid dims
-            int search_block_size   = INIT_KERNEL_THREADS;
-            int search_grid_size    = (num_merge_tiles + 1 + search_block_size - 1) / search_block_size;
-
-#if (CUB_PTX_ARCH == 0)
-            // Init textures
-            if (CubDebug(error = spmv_params.t_vector_x.BindTexture(spmv_params.d_vector_x))) break;
-#endif
-
-            if (search_grid_size < sm_count)
-//            if (num_merge_tiles < spmv_sm_occupancy * sm_count)
-            {
-                // Not enough spmv tiles to saturate the device: have spmv blocks search their own staring coords
-                d_tile_coordinates = NULL;
-            }
-            else
-            {
-                // Use separate search kernel if we have enough spmv tiles to saturate the device
-
-                // Log spmv_search_kernel configuration
-                if (debug_synchronous) _CubLog("Invoking spmv_search_kernel<<<%d, %d, 0, %lld>>>()\n",
-                    search_grid_size, search_block_size, (long long) stream);
-
-                // Invoke spmv_search_kernel
-                spmv_search_kernel<<<search_grid_size, search_block_size, 0, stream>>>(
-                    num_merge_tiles,
-                    d_tile_coordinates,
-                    spmv_params);
-
-                // Check for failure to launch
-                if (CubDebug(error = cudaPeekAtLastError())) break;
-
-                // Sync the stream if specified to flush runtime errors
-                if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-            }
-
-            // Log spmv_kernel configuration
-            if (debug_synchronous) _CubLog("Invoking spmv_kernel<<<{%d,%d,%d}, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
-                spmv_grid_size.x, spmv_grid_size.y, spmv_grid_size.z, spmv_config.block_threads, (long long) stream, spmv_config.items_per_thread, spmv_sm_occupancy);
-
-            // Invoke spmv_kernel
-            spmv_kernel<<<spmv_grid_size, spmv_config.block_threads, 0, stream>>>(
-                spmv_params,
-                d_tile_coordinates,
-                d_tile_carry_pairs,
-                num_merge_tiles,
-                tile_state,
-                num_segment_fixup_tiles);
-
-            // Check for failure to launch
-            if (CubDebug(error = cudaPeekAtLastError())) break;
-
-            // Sync the stream if specified to flush runtime errors
-            if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-
-            // Run reduce-by-key fixup if necessary
-            if (num_merge_tiles > 1)
-            {
-                // Log segment_fixup_kernel configuration
-                if (debug_synchronous) _CubLog("Invoking segment_fixup_kernel<<<{%d,%d,%d}, %d, 0, %lld>>>(), %d items per thread, %d SM occupancy\n",
-                    segment_fixup_grid_size.x, segment_fixup_grid_size.y, segment_fixup_grid_size.z, segment_fixup_config.block_threads, (long long) stream, segment_fixup_config.items_per_thread, segment_fixup_sm_occupancy);
-
-                // Invoke segment_fixup_kernel
-                segment_fixup_kernel<<<segment_fixup_grid_size, segment_fixup_config.block_threads, 0, stream>>>(
-                    d_tile_carry_pairs,
-                    spmv_params.d_vector_y,
-                    num_merge_tiles,
-                    num_segment_fixup_tiles,
-                    tile_state);
-
-                // Check for failure to launch
-                if (CubDebug(error = cudaPeekAtLastError())) break;
-
-                // Sync the stream if specified to flush runtime errors
-                if (debug_synchronous && (CubDebug(error = SyncStream(stream)))) break;
-            }
-
-#if (CUB_PTX_ARCH == 0)
-            // Free textures
-            if (CubDebug(error = spmv_params.t_vector_x.UnbindTexture())) break;
-#endif
-        }
-        while (0);
-
-        return error;
-
-#endif // CUB_RUNTIME_ENABLED
-    }
-
-
-    /**
-     * Internal dispatch routine for computing a device-wide reduction
-     */
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Dispatch(
-        void*                   d_temp_storage,                     ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-        size_t&                 temp_storage_bytes,                 ///< [in,out] Reference to size in bytes of \p d_temp_storage allocation
-        SpmvParamsT&            spmv_params,                        ///< SpMV input parameter bundle
-        cudaStream_t            stream                  = 0,        ///< [in] <b>[optional]</b> CUDA stream to launch kernels within.  Default is stream<sub>0</sub>.
-        bool                    debug_synchronous       = false)    ///< [in] <b>[optional]</b> Whether or not to synchronize the stream after every kernel launch to check for errors.  May cause significant slowdown.  Default is \p false.
-    {
-        cudaError error = cudaSuccess;
-        do
-        {
-            // Get PTX version
-            int ptx_version;
-    #if (CUB_PTX_ARCH == 0)
-            if (CubDebug(error = PtxVersion(ptx_version))) break;
-    #else
-            ptx_version = CUB_PTX_ARCH;
-    #endif
-
-            // Get kernel kernel dispatch configurations
-            KernelConfig spmv_config, segment_fixup_config;
-            InitConfigs(ptx_version, spmv_config, segment_fixup_config);
-
-            if (CubDebug(error = Dispatch(
-                d_temp_storage, temp_storage_bytes, spmv_params, stream, debug_synchronous,
-                DeviceSpmv1ColKernel<PtxSpmvPolicyT, ValueT, OffsetT>,
-                DeviceSpmvSearchKernel<PtxSpmvPolicyT, OffsetT, CoordinateT, SpmvParamsT>,
-                DeviceSpmvKernel<PtxSpmvPolicyT, ScanTileStateT, ValueT, OffsetT, CoordinateT, false, false>,
-                DeviceSegmentFixupKernel<PtxSegmentFixupPolicy, KeyValuePairT*, ValueT*, OffsetT, ScanTileStateT>,
-                spmv_config, segment_fixup_config))) break;
-
-        }
-        while (0);
-
-        return error;
-    }
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/grid/grid_barrier.cuh b/pfsimulator/third_party/cub/grid/grid_barrier.cuh
deleted file mode 100644
index 461fb4421..000000000
--- a/pfsimulator/third_party/cub/grid/grid_barrier.cuh
+++ /dev/null
@@ -1,211 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::GridBarrier implements a software global barrier among thread blocks within a CUDA grid
- */
-
-#pragma once
-
-#include "../util_debug.cuh"
-#include "../util_namespace.cuh"
-#include "../thread/thread_load.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \addtogroup GridModule
- * @{
- */
-
-
-/**
- * \brief GridBarrier implements a software global barrier among thread blocks within a CUDA grid
- */
-class GridBarrier
-{
-protected :
-
-    typedef unsigned int SyncFlag;
-
-    // Counters in global device memory
-    SyncFlag* d_sync;
-
-public:
-
-    /**
-     * Constructor
-     */
-    GridBarrier() : d_sync(NULL) {}
-
-
-    /**
-     * Synchronize
-     */
-    __device__ __forceinline__ void Sync() const
-    {
-        volatile SyncFlag *d_vol_sync = d_sync;
-
-        // Threadfence and syncthreads to make sure global writes are visible before
-        // thread-0 reports in with its sync counter
-        __threadfence();
-        CTA_SYNC();
-
-        if (blockIdx.x == 0)
-        {
-            // Report in ourselves
-            if (threadIdx.x == 0)
-            {
-                d_vol_sync[blockIdx.x] = 1;
-            }
-
-            CTA_SYNC();
-
-            // Wait for everyone else to report in
-            for (int peer_block = threadIdx.x; peer_block < gridDim.x; peer_block += blockDim.x)
-            {
-                while (ThreadLoad<LOAD_CG>(d_sync + peer_block) == 0)
-                {
-                    __threadfence_block();
-                }
-            }
-
-            CTA_SYNC();
-
-            // Let everyone know it's safe to proceed
-            for (int peer_block = threadIdx.x; peer_block < gridDim.x; peer_block += blockDim.x)
-            {
-                d_vol_sync[peer_block] = 0;
-            }
-        }
-        else
-        {
-            if (threadIdx.x == 0)
-            {
-                // Report in
-                d_vol_sync[blockIdx.x] = 1;
-
-                // Wait for acknowledgment
-                while (ThreadLoad<LOAD_CG>(d_sync + blockIdx.x) == 1)
-                {
-                    __threadfence_block();
-                }
-            }
-
-            CTA_SYNC();
-        }
-    }
-};
-
-
-/**
- * \brief GridBarrierLifetime extends GridBarrier to provide lifetime management of the temporary device storage needed for cooperation.
- *
- * Uses RAII for lifetime, i.e., device resources are reclaimed when
- * the destructor is called.
- */
-class GridBarrierLifetime : public GridBarrier
-{
-protected:
-
-    // Number of bytes backed by d_sync
-    size_t sync_bytes;
-
-public:
-
-    /**
-     * Constructor
-     */
-    GridBarrierLifetime() : GridBarrier(), sync_bytes(0) {}
-
-
-    /**
-     * DeviceFrees and resets the progress counters
-     */
-    cudaError_t HostReset()
-    {
-        cudaError_t retval = cudaSuccess;
-        if (d_sync)
-        {
-            CubDebug(retval = cudaFree(d_sync));
-            d_sync = NULL;
-        }
-        sync_bytes = 0;
-        return retval;
-    }
-
-
-    /**
-     * Destructor
-     */
-    virtual ~GridBarrierLifetime()
-    {
-        HostReset();
-    }
-
-
-    /**
-     * Sets up the progress counters for the next kernel launch (lazily
-     * allocating and initializing them if necessary)
-     */
-    cudaError_t Setup(int sweep_grid_size)
-    {
-        cudaError_t retval = cudaSuccess;
-        do {
-            size_t new_sync_bytes = sweep_grid_size * sizeof(SyncFlag);
-            if (new_sync_bytes > sync_bytes)
-            {
-                if (d_sync)
-                {
-                    if (CubDebug(retval = cudaFree(d_sync))) break;
-                }
-
-                sync_bytes = new_sync_bytes;
-
-                // Allocate and initialize to zero
-                if (CubDebug(retval = cudaMalloc((void**) &d_sync, sync_bytes))) break;
-                if (CubDebug(retval = cudaMemset(d_sync, 0, new_sync_bytes))) break;
-            }
-        } while (0);
-
-        return retval;
-    }
-};
-
-
-/** @} */       // end group GridModule
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/grid/grid_even_share.cuh b/pfsimulator/third_party/cub/grid/grid_even_share.cuh
deleted file mode 100644
index f0b3a69ae..000000000
--- a/pfsimulator/third_party/cub/grid/grid_even_share.cuh
+++ /dev/null
@@ -1,222 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::GridEvenShare is a descriptor utility for distributing input among CUDA thread blocks in an "even-share" fashion.  Each thread block gets roughly the same number of fixed-size work units (grains).
- */
-
-
-#pragma once
-
-#include "../util_namespace.cuh"
-#include "../util_macro.cuh"
-#include "grid_mapping.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \addtogroup GridModule
- * @{
- */
-
-
-/**
- * \brief GridEvenShare is a descriptor utility for distributing input among
- * CUDA thread blocks in an "even-share" fashion.  Each thread block gets roughly
- * the same number of input tiles.
- *
- * \par Overview
- * Each thread block is assigned a consecutive sequence of input tiles.  To help
- * preserve alignment and eliminate the overhead of guarded loads for all but the
- * last thread block, to GridEvenShare assigns one of three different amounts of
- * work to a given thread block: "big", "normal", or "last".  The "big" workloads
- * are one scheduling grain larger than "normal".  The "last" work unit for the
- * last thread block may be partially-full if the input is not an even multiple of
- * the scheduling grain size.
- *
- * \par
- * Before invoking a child grid, a parent thread will typically construct an
- * instance of GridEvenShare.  The instance can be passed to child thread blocks
- * which can initialize their per-thread block offsets using \p BlockInit().
- */
-template <typename OffsetT>
-struct GridEvenShare
-{
-private:
-
-    OffsetT     total_tiles;
-    int         big_shares;
-    OffsetT     big_share_items;
-    OffsetT     normal_share_items;
-    OffsetT     normal_base_offset;
-
-public:
-
-    /// Total number of input items
-    OffsetT     num_items;
-
-    /// Grid size in thread blocks
-    int         grid_size;
-
-    /// OffsetT into input marking the beginning of the owning thread block's segment of input tiles
-    OffsetT     block_offset;
-
-    /// OffsetT into input of marking the end (one-past) of the owning thread block's segment of input tiles
-    OffsetT     block_end;
-
-    /// Stride between input tiles
-    OffsetT     block_stride;
-
-
-    /**
-     * \brief Constructor.
-     */
-    __host__ __device__ __forceinline__ GridEvenShare() :
-        total_tiles(0),
-        big_shares(0),
-        big_share_items(0),
-        normal_share_items(0),
-        normal_base_offset(0),
-        num_items(0),
-        grid_size(0),
-        block_offset(0),
-        block_end(0),
-        block_stride(0)
-    {}
-
-
-    /**
-     * \brief Dispatch initializer. To be called prior prior to kernel launch.
-     */
-    __host__ __device__ __forceinline__ void DispatchInit(
-        OffsetT num_items,          ///< Total number of input items
-        int     max_grid_size,      ///< Maximum grid size allowable (actual grid size may be less if not warranted by the the number of input items)
-        int     tile_items)         ///< Number of data items per input tile
-    {
-        this->block_offset          = num_items;    // Initialize past-the-end
-        this->block_end             = num_items;    // Initialize past-the-end
-        this->num_items             = num_items;
-        this->total_tiles           = (num_items + tile_items - 1) / tile_items;
-        this->grid_size             = CUB_MIN(total_tiles, max_grid_size);
-        OffsetT avg_tiles_per_block = total_tiles / grid_size;
-        this->big_shares            = total_tiles - (avg_tiles_per_block * grid_size);        // leftover grains go to big blocks
-        this->normal_share_items    = avg_tiles_per_block * tile_items;
-        this->normal_base_offset    = big_shares * tile_items;
-        this->big_share_items       = normal_share_items + tile_items;
-    }
-
-
-    /**
-     * \brief Initializes ranges for the specified thread block index.  Specialized
-     * for a "raking" access pattern in which each thread block is assigned a
-     * consecutive sequence of input tiles.
-     */
-    template <int TILE_ITEMS>
-    __device__ __forceinline__ void BlockInit(
-        int block_id,
-        Int2Type<GRID_MAPPING_RAKE> /*strategy_tag*/)
-    {
-        block_stride = TILE_ITEMS;
-        if (block_id < big_shares)
-        {
-            // This thread block gets a big share of grains (avg_tiles_per_block + 1)
-            block_offset = (block_id * big_share_items);
-            block_end = block_offset + big_share_items;
-        }
-        else if (block_id < total_tiles)
-        {
-            // This thread block gets a normal share of grains (avg_tiles_per_block)
-            block_offset = normal_base_offset + (block_id * normal_share_items);
-            block_end = CUB_MIN(num_items, block_offset + normal_share_items);
-        }
-        // Else default past-the-end
-    }
-
-
-    /**
-     * \brief Block-initialization, specialized for a "raking" access
-     * pattern in which each thread block is assigned a consecutive sequence
-     * of input tiles.
-     */
-    template <int TILE_ITEMS>
-    __device__ __forceinline__ void BlockInit(
-        int block_id,
-        Int2Type<GRID_MAPPING_STRIP_MINE> /*strategy_tag*/)
-    {
-        block_stride = grid_size * TILE_ITEMS;
-        block_offset = (block_id * TILE_ITEMS);
-        block_end = num_items;
-    }
-
-
-    /**
-     * \brief Block-initialization, specialized for "strip mining" access
-     * pattern in which the input tiles assigned to each thread block are
-     * separated by a stride equal to the the extent of the grid.
-     */
-    template <
-        int TILE_ITEMS,
-        GridMappingStrategy STRATEGY>
-    __device__ __forceinline__ void BlockInit()
-    {
-        BlockInit<TILE_ITEMS>(blockIdx.x, Int2Type<STRATEGY>());
-    }
-
-
-    /**
-     * \brief Block-initialization, specialized for a "raking" access
-     * pattern in which each thread block is assigned a consecutive sequence
-     * of input tiles.
-     */
-    template <int TILE_ITEMS>
-    __device__ __forceinline__ void BlockInit(
-        OffsetT block_offset,                       ///< [in] Threadblock begin offset (inclusive)
-        OffsetT block_end)                          ///< [in] Threadblock end offset (exclusive)
-    {
-        this->block_offset = block_offset;
-        this->block_end = block_end;
-        this->block_stride = TILE_ITEMS;
-    }
-
-
-};
-
-
-
-
-
-/** @} */       // end group GridModule
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/grid/grid_mapping.cuh b/pfsimulator/third_party/cub/grid/grid_mapping.cuh
deleted file mode 100644
index f0e9fded2..000000000
--- a/pfsimulator/third_party/cub/grid/grid_mapping.cuh
+++ /dev/null
@@ -1,113 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::GridMappingStrategy enumerates alternative strategies for mapping constant-sized tiles of device-wide data onto a grid of CUDA thread blocks.
- */
-
-#pragma once
-
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \addtogroup GridModule
- * @{
- */
-
-
-/******************************************************************************
- * Mapping policies
- *****************************************************************************/
-
-
-/**
- * \brief cub::GridMappingStrategy enumerates alternative strategies for mapping constant-sized tiles of device-wide data onto a grid of CUDA thread blocks.
- */
-enum GridMappingStrategy
-{
-    /**
-     * \brief An a "raking" access pattern in which each thread block is
-     * assigned a consecutive sequence of input tiles
-     *
-     * \par Overview
-     * The input is evenly partitioned into \p p segments, where \p p is
-     * constant and corresponds loosely to the number of thread blocks that may
-     * actively reside on the target device. Each segment is comprised of
-     * consecutive tiles, where a tile is a small, constant-sized unit of input
-     * to be processed to completion before the thread block terminates or
-     * obtains more work.  The kernel invokes \p p thread blocks, each
-     * of which iteratively consumes a segment of <em>n</em>/<em>p</em> elements
-     * in tile-size increments.
-     */
-    GRID_MAPPING_RAKE,
-
-    /**
-     * \brief An a "strip mining" access pattern in which the input tiles assigned
-     * to each thread block are separated by a stride equal to the the extent of
-     * the grid.
-     *
-     * \par Overview
-     * The input is evenly partitioned into \p p sets, where \p p is
-     * constant and corresponds loosely to the number of thread blocks that may
-     * actively reside on the target device. Each set is comprised of
-     * data tiles separated by stride \p tiles, where a tile is a small,
-     * constant-sized unit of input to be processed to completion before the
-     * thread block terminates or obtains more work.  The kernel invokes \p p
-     * thread blocks, each of which iteratively consumes a segment of
-     * <em>n</em>/<em>p</em> elements in tile-size increments.
-     */
-    GRID_MAPPING_STRIP_MINE,
-
-    /**
-     * \brief A dynamic "queue-based" strategy for assigning input tiles to thread blocks.
-     *
-     * \par Overview
-     * The input is treated as a queue to be dynamically consumed by a grid of
-     * thread blocks.  Work is atomically dequeued in tiles, where a tile is a
-     * unit of input to be processed to completion before the thread block
-     * terminates or obtains more work.  The grid size \p p is constant,
-     * loosely corresponding to the number of thread blocks that may actively
-     * reside on the target device.
-     */
-    GRID_MAPPING_DYNAMIC,
-};
-
-
-/** @} */       // end group GridModule
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/grid/grid_queue.cuh b/pfsimulator/third_party/cub/grid/grid_queue.cuh
deleted file mode 100644
index 9615b14db..000000000
--- a/pfsimulator/third_party/cub/grid/grid_queue.cuh
+++ /dev/null
@@ -1,220 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::GridQueue is a descriptor utility for dynamic queue management.
- */
-
-#pragma once
-
-#include "../util_namespace.cuh"
-#include "../util_debug.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \addtogroup GridModule
- * @{
- */
-
-
-/**
- * \brief GridQueue is a descriptor utility for dynamic queue management.
- *
- * \par Overview
- * GridQueue descriptors provides abstractions for "filling" or
- * "draining" globally-shared vectors.
- *
- * \par
- * A "filling" GridQueue works by atomically-adding to a zero-initialized counter,
- * returning a unique offset for the calling thread to write its items.
- * The GridQueue maintains the total "fill-size".  The fill counter must be reset
- * using GridQueue::ResetFill by the host or kernel instance prior to the kernel instance that
- * will be filling.
- *
- * \par
- * Similarly, a "draining" GridQueue works by works by atomically-incrementing a
- * zero-initialized counter, returning a unique offset for the calling thread to
- * read its items. Threads can safely drain until the array's logical fill-size is
- * exceeded.  The drain counter must be reset using GridQueue::ResetDrain or
- * GridQueue::FillAndResetDrain by the host or kernel instance prior to the kernel instance that
- * will be filling.  (For dynamic work distribution of existing data, the corresponding fill-size
- * is simply the number of elements in the array.)
- *
- * \par
- * Iterative work management can be implemented simply with a pair of flip-flopping
- * work buffers, each with an associated set of fill and drain GridQueue descriptors.
- *
- * \tparam OffsetT Signed integer type for global offsets
- */
-template <typename OffsetT>
-class GridQueue
-{
-private:
-
-    /// Counter indices
-    enum
-    {
-        FILL    = 0,
-        DRAIN   = 1,
-    };
-
-    /// Pair of counters
-    OffsetT *d_counters;
-
-public:
-
-    /// Returns the device allocation size in bytes needed to construct a GridQueue instance
-    __host__ __device__ __forceinline__
-    static size_t AllocationSize()
-    {
-        return sizeof(OffsetT) * 2;
-    }
-
-
-    /// Constructs an invalid GridQueue descriptor
-    __host__ __device__ __forceinline__ GridQueue()
-    :
-        d_counters(NULL)
-    {}
-
-
-    /// Constructs a GridQueue descriptor around the device storage allocation
-    __host__ __device__ __forceinline__ GridQueue(
-        void *d_storage)                    ///< Device allocation to back the GridQueue.  Must be at least as big as <tt>AllocationSize()</tt>.
-    :
-        d_counters((OffsetT*) d_storage)
-    {}
-
-
-    /// This operation sets the fill-size and resets the drain counter, preparing the GridQueue for draining in the next kernel instance.  To be called by the host or by a kernel prior to that which will be draining.
-    __host__ __device__ __forceinline__ cudaError_t FillAndResetDrain(
-        OffsetT fill_size,
-        cudaStream_t stream = 0)
-    {
-#if (CUB_PTX_ARCH > 0)
-        (void)stream;
-        d_counters[FILL] = fill_size;
-        d_counters[DRAIN] = 0;
-        return cudaSuccess;
-#else
-        OffsetT counters[2];
-        counters[FILL] = fill_size;
-        counters[DRAIN] = 0;
-        return CubDebug(cudaMemcpyAsync(d_counters, counters, sizeof(OffsetT) * 2, cudaMemcpyHostToDevice, stream));
-#endif
-    }
-
-
-    /// This operation resets the drain so that it may advance to meet the existing fill-size.  To be called by the host or by a kernel prior to that which will be draining.
-    __host__ __device__ __forceinline__ cudaError_t ResetDrain(cudaStream_t stream = 0)
-    {
-#if (CUB_PTX_ARCH > 0)
-        (void)stream;
-        d_counters[DRAIN] = 0;
-        return cudaSuccess;
-#else
-        return CubDebug(cudaMemsetAsync(d_counters + DRAIN, 0, sizeof(OffsetT), stream));
-#endif
-    }
-
-
-    /// This operation resets the fill counter.  To be called by the host or by a kernel prior to that which will be filling.
-    __host__ __device__ __forceinline__ cudaError_t ResetFill(cudaStream_t stream = 0)
-    {
-#if (CUB_PTX_ARCH > 0)
-        (void)stream;
-        d_counters[FILL] = 0;
-        return cudaSuccess;
-#else
-        return CubDebug(cudaMemsetAsync(d_counters + FILL, 0, sizeof(OffsetT), stream));
-#endif
-    }
-
-
-    /// Returns the fill-size established by the parent or by the previous kernel.
-    __host__ __device__ __forceinline__ cudaError_t FillSize(
-        OffsetT &fill_size,
-        cudaStream_t stream = 0)
-    {
-#if (CUB_PTX_ARCH > 0)
-        (void)stream;
-        fill_size = d_counters[FILL];
-        return cudaSuccess;
-#else
-        return CubDebug(cudaMemcpyAsync(&fill_size, d_counters + FILL, sizeof(OffsetT), cudaMemcpyDeviceToHost, stream));
-#endif
-    }
-
-
-    /// Drain \p num_items from the queue.  Returns offset from which to read items.  To be called from CUDA kernel.
-    __device__ __forceinline__ OffsetT Drain(OffsetT num_items)
-    {
-        return atomicAdd(d_counters + DRAIN, num_items);
-    }
-
-
-    /// Fill \p num_items into the queue.  Returns offset from which to write items.    To be called from CUDA kernel.
-    __device__ __forceinline__ OffsetT Fill(OffsetT num_items)
-    {
-        return atomicAdd(d_counters + FILL, num_items);
-    }
-};
-
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-
-/**
- * Reset grid queue (call with 1 block of 1 thread)
- */
-template <typename OffsetT>
-__global__ void FillAndResetDrainKernel(
-    GridQueue<OffsetT>   grid_queue,
-    OffsetT              num_items)
-{
-    grid_queue.FillAndResetDrain(num_items);
-}
-
-
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-/** @} */       // end group GridModule
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-
diff --git a/pfsimulator/third_party/cub/host/mutex.cuh b/pfsimulator/third_party/cub/host/mutex.cuh
deleted file mode 100644
index ff7ec90dd..000000000
--- a/pfsimulator/third_party/cub/host/mutex.cuh
+++ /dev/null
@@ -1,171 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Simple portable mutex
- */
-
-
-#pragma once
-
-#if (__cplusplus > 199711L) || (defined(_MSC_VER) && _MSC_VER >= 1800)
-    #include <mutex>
-#else
-    #if defined(_WIN32) || defined(_WIN64)
-        #include <intrin.h>
-
-        #define WIN32_LEAN_AND_MEAN
-        #define NOMINMAX
-        #include <windows.h>
-        #undef WIN32_LEAN_AND_MEAN
-        #undef NOMINMAX
-
-        /**
-         * Compiler read/write barrier
-         */
-        #pragma intrinsic(_ReadWriteBarrier)
-
-    #endif
-#endif
-
-#include "../util_namespace.cuh"
-
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * Simple portable mutex
- *   - Wraps std::mutex when compiled with C++11 or newer (supported on all platforms)
- *   - Uses GNU/Windows spinlock mechanisms for pre C++11 (supported on x86/x64 when compiled with cl.exe or g++)
- */
-struct Mutex
-{
-#if (__cplusplus > 199711L) || (defined(_MSC_VER) && _MSC_VER >= 1800)
-
-    std::mutex mtx;
-
-    void Lock()
-    {
-        mtx.lock();
-    }
-
-    void Unlock()
-    {
-        mtx.unlock();
-    }
-
-    void TryLock()
-    {
-        mtx.try_lock();
-    }
-
-#else       //__cplusplus > 199711L
-
-    #if defined(_MSC_VER)
-
-        // Microsoft VC++
-        typedef long Spinlock;
-
-    #else
-
-        // GNU g++
-        typedef int Spinlock;
-
-        /**
-         * Compiler read/write barrier
-         */
-        __forceinline__ void _ReadWriteBarrier()
-        {
-            __sync_synchronize();
-        }
-
-        /**
-         * Atomic exchange
-         */
-        __forceinline__ long _InterlockedExchange(volatile int * const Target, const int Value)
-        {
-            // NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier
-            _ReadWriteBarrier();
-            return __sync_lock_test_and_set(Target, Value);
-        }
-
-        /**
-         * Pause instruction to prevent excess processor bus usage
-         */
-        __forceinline__ void YieldProcessor()
-        {
-        }
-
-    #endif  // defined(_MSC_VER)
-
-        /// Lock member
-        volatile Spinlock lock;
-
-        /**
-         * Constructor
-         */
-        Mutex() : lock(0) {}
-
-        /**
-         * Return when the specified spinlock has been acquired
-         */
-        __forceinline__ void Lock()
-        {
-            while (1)
-            {
-                if (!_InterlockedExchange(&lock, 1)) return;
-                while (lock) YieldProcessor();
-            }
-        }
-
-
-        /**
-         * Release the specified spinlock
-         */
-        __forceinline__ void Unlock()
-        {
-            _ReadWriteBarrier();
-            lock = 0;
-        }
-
-#endif      // __cplusplus > 199711L
-
-};
-
-
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
diff --git a/pfsimulator/third_party/cub/iterator/arg_index_input_iterator.cuh b/pfsimulator/third_party/cub/iterator/arg_index_input_iterator.cuh
deleted file mode 100644
index 95a84a579..000000000
--- a/pfsimulator/third_party/cub/iterator/arg_index_input_iterator.cuh
+++ /dev/null
@@ -1,259 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Random-access iterator types
- */
-
-#pragma once
-
-#include <iterator>
-#include <iostream>
-
-#include "../thread/thread_load.cuh"
-#include "../thread/thread_store.cuh"
-#include "../util_device.cuh"
-#include "../util_namespace.cuh"
-
-#include <thrust/version.h>
-
-#if (THRUST_VERSION >= 100700)
-    // This iterator is compatible with Thrust API 1.7 and newer
-    #include <thrust/iterator/iterator_facade.h>
-    #include <thrust/iterator/iterator_traits.h>
-#endif // THRUST_VERSION
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \addtogroup UtilIterator
- * @{
- */
-
-
-/**
- * \brief A random-access input wrapper for pairing dereferenced values with their corresponding indices (forming \p KeyValuePair tuples).
- *
- * \par Overview
- * - ArgIndexInputIteratorTwraps a random access input iterator \p itr of type \p InputIteratorT.
- *   Dereferencing an ArgIndexInputIteratorTat offset \p i produces a \p KeyValuePair value whose
- *   \p key field is \p i and whose \p value field is <tt>itr[i]</tt>.
- * - Can be used with any data type.
- * - Can be constructed, manipulated, and exchanged within and between host and device
- *   functions.  Wrapped host memory can only be dereferenced on the host, and wrapped
- *   device memory can only be dereferenced on the device.
- * - Compatible with Thrust API v1.7 or newer.
- *
- * \par Snippet
- * The code snippet below illustrates the use of \p ArgIndexInputIteratorTto
- * dereference an array of doubles
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/iterator/arg_index_input_iterator.cuh>
- *
- * // Declare, allocate, and initialize a device array
- * double *d_in;         // e.g., [8.0, 6.0, 7.0, 5.0, 3.0, 0.0, 9.0]
- *
- * // Create an iterator wrapper
- * cub::ArgIndexInputIterator<double*> itr(d_in);
- *
- * // Within device code:
- * typedef typename cub::ArgIndexInputIterator<double*>::value_type Tuple;
- * Tuple item_offset_pair.key = *itr;
- * printf("%f @ %d\n",
- *   item_offset_pair.value,
- *   item_offset_pair.key);   // 8.0 @ 0
- *
- * itr = itr + 6;
- * item_offset_pair.key = *itr;
- * printf("%f @ %d\n",
- *   item_offset_pair.value,
- *   item_offset_pair.key);   // 9.0 @ 6
- *
- * \endcode
- *
- * \tparam InputIteratorT       The value type of the wrapped input iterator
- * \tparam OffsetT              The difference type of this iterator (Default: \p ptrdiff_t)
- * \tparam OutputValueT         The paired value type of the <offset,value> tuple (Default: value type of input iterator)
- */
-template <
-    typename    InputIteratorT,
-    typename    OffsetT             = ptrdiff_t,
-    typename    OutputValueT        = typename std::iterator_traits<InputIteratorT>::value_type>
-class ArgIndexInputIterator
-{
-public:
-
-    // Required iterator traits
-    typedef ArgIndexInputIterator                       self_type;              ///< My own type
-    typedef OffsetT                                     difference_type;        ///< Type to express the result of subtracting one iterator from another
-    typedef KeyValuePair<difference_type, OutputValueT> value_type;             ///< The type of the element the iterator can point to
-    typedef value_type*                                 pointer;                ///< The type of a pointer to an element the iterator can point to
-    typedef value_type                                  reference;              ///< The type of a reference to an element the iterator can point to
-
-#if (THRUST_VERSION >= 100700)
-    // Use Thrust's iterator categories so we can use these iterators in Thrust 1.7 (or newer) methods
-    typedef typename thrust::detail::iterator_facade_category<
-        thrust::any_system_tag,
-        thrust::random_access_traversal_tag,
-        value_type,
-        reference
-      >::type iterator_category;                                        ///< The iterator category
-#else
-    typedef std::random_access_iterator_tag     iterator_category;      ///< The iterator category
-#endif  // THRUST_VERSION
-
-private:
-
-    InputIteratorT  itr;
-    difference_type offset;
-
-public:
-
-    /// Constructor
-    __host__ __device__ __forceinline__ ArgIndexInputIterator(
-        InputIteratorT  itr,            ///< Input iterator to wrap
-        difference_type offset = 0)     ///< OffsetT (in items) from \p itr denoting the position of the iterator
-    :
-        itr(itr),
-        offset(offset)
-    {}
-
-    /// Postfix increment
-    __host__ __device__ __forceinline__ self_type operator++(int)
-    {
-        self_type retval = *this;
-        offset++;
-        return retval;
-    }
-
-    /// Prefix increment
-    __host__ __device__ __forceinline__ self_type operator++()
-    {
-        offset++;
-        return *this;
-    }
-
-    /// Indirection
-    __host__ __device__ __forceinline__ reference operator*() const
-    {
-        value_type retval;
-        retval.value = itr[offset];
-        retval.key = offset;
-        return retval;
-    }
-
-    /// Addition
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type operator+(Distance n) const
-    {
-        self_type retval(itr, offset + n);
-        return retval;
-    }
-
-    /// Addition assignment
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator+=(Distance n)
-    {
-        offset += n;
-        return *this;
-    }
-
-    /// Subtraction
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type operator-(Distance n) const
-    {
-        self_type retval(itr, offset - n);
-        return retval;
-    }
-
-    /// Subtraction assignment
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator-=(Distance n)
-    {
-        offset -= n;
-        return *this;
-    }
-
-    /// Distance
-    __host__ __device__ __forceinline__ difference_type operator-(self_type other) const
-    {
-        return offset - other.offset;
-    }
-
-    /// Array subscript
-    template <typename Distance>
-    __host__ __device__ __forceinline__ reference operator[](Distance n) const
-    {
-        self_type offset = (*this) + n;
-        return *offset;
-    }
-
-    /// Structure dereference
-    __host__ __device__ __forceinline__ pointer operator->()
-    {
-        return &(*(*this));
-    }
-
-    /// Equal to
-    __host__ __device__ __forceinline__ bool operator==(const self_type& rhs)
-    {
-        return ((itr == rhs.itr) && (offset == rhs.offset));
-    }
-
-    /// Not equal to
-    __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs)
-    {
-        return ((itr != rhs.itr) || (offset != rhs.offset));
-    }
-
-    /// Normalize
-    __host__ __device__ __forceinline__ void normalize()
-    {
-        itr += offset;
-        offset = 0;
-    }
-
-    /// ostream operator
-    friend std::ostream& operator<<(std::ostream& os, const self_type& /*itr*/)
-    {
-        return os;
-    }
-};
-
-
-
-/** @} */       // end group UtilIterator
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/iterator/cache_modified_input_iterator.cuh b/pfsimulator/third_party/cub/iterator/cache_modified_input_iterator.cuh
deleted file mode 100644
index b4ad91e2f..000000000
--- a/pfsimulator/third_party/cub/iterator/cache_modified_input_iterator.cuh
+++ /dev/null
@@ -1,240 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Random-access iterator types
- */
-
-#pragma once
-
-#include <iterator>
-#include <iostream>
-
-#include "../thread/thread_load.cuh"
-#include "../thread/thread_store.cuh"
-#include "../util_device.cuh"
-#include "../util_namespace.cuh"
-
-#if (THRUST_VERSION >= 100700)
-    // This iterator is compatible with Thrust API 1.7 and newer
-    #include <thrust/iterator/iterator_facade.h>
-    #include <thrust/iterator/iterator_traits.h>
-#endif // THRUST_VERSION
-
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-
-/**
- * \addtogroup UtilIterator
- * @{
- */
-
-
-/**
- * \brief A random-access input wrapper for dereferencing array values using a PTX cache load modifier.
- *
- * \par Overview
- * - CacheModifiedInputIteratorTis a random-access input iterator that wraps a native
- *   device pointer of type <tt>ValueType*</tt>. \p ValueType references are
- *   made by reading \p ValueType values through loads modified by \p MODIFIER.
- * - Can be used to load any data type from memory using PTX cache load modifiers (e.g., "LOAD_LDG",
- *   "LOAD_CG", "LOAD_CA", "LOAD_CS", "LOAD_CV", etc.).
- * - Can be constructed, manipulated, and exchanged within and between host and device
- *   functions, but can only be dereferenced within device functions.
- * - Compatible with Thrust API v1.7 or newer.
- *
- * \par Snippet
- * The code snippet below illustrates the use of \p CacheModifiedInputIteratorTto
- * dereference a device array of double using the "ldg" PTX load modifier
- * (i.e., load values through texture cache).
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/iterator/cache_modified_input_iterator.cuh>
- *
- * // Declare, allocate, and initialize a device array
- * double *d_in;            // e.g., [8.0, 6.0, 7.0, 5.0, 3.0, 0.0, 9.0]
- *
- * // Create an iterator wrapper
- * cub::CacheModifiedInputIterator<cub::LOAD_LDG, double> itr(d_in);
- *
- * // Within device code:
- * printf("%f\n", itr[0]);  // 8.0
- * printf("%f\n", itr[1]);  // 6.0
- * printf("%f\n", itr[6]);  // 9.0
- *
- * \endcode
- *
- * \tparam CacheLoadModifier    The cub::CacheLoadModifier to use when accessing data
- * \tparam ValueType            The value type of this iterator
- * \tparam OffsetT              The difference type of this iterator (Default: \p ptrdiff_t)
- */
-template <
-    CacheLoadModifier   MODIFIER,
-    typename            ValueType,
-    typename            OffsetT = ptrdiff_t>
-class CacheModifiedInputIterator
-{
-public:
-
-    // Required iterator traits
-    typedef CacheModifiedInputIterator          self_type;              ///< My own type
-    typedef OffsetT                             difference_type;        ///< Type to express the result of subtracting one iterator from another
-    typedef ValueType                           value_type;             ///< The type of the element the iterator can point to
-    typedef ValueType*                          pointer;                ///< The type of a pointer to an element the iterator can point to
-    typedef ValueType                           reference;              ///< The type of a reference to an element the iterator can point to
-
-#if (THRUST_VERSION >= 100700)
-    // Use Thrust's iterator categories so we can use these iterators in Thrust 1.7 (or newer) methods
-    typedef typename thrust::detail::iterator_facade_category<
-        thrust::device_system_tag,
-        thrust::random_access_traversal_tag,
-        value_type,
-        reference
-      >::type iterator_category;                                        ///< The iterator category
-#else
-    typedef std::random_access_iterator_tag     iterator_category;      ///< The iterator category
-#endif  // THRUST_VERSION
-
-
-public:
-
-    /// Wrapped native pointer
-    ValueType* ptr;
-
-    /// Constructor
-    template <typename QualifiedValueType>
-    __host__ __device__ __forceinline__ CacheModifiedInputIterator(
-        QualifiedValueType* ptr)     ///< Native pointer to wrap
-    :
-        ptr(const_cast<typename RemoveQualifiers<QualifiedValueType>::Type *>(ptr))
-    {}
-
-    /// Postfix increment
-    __host__ __device__ __forceinline__ self_type operator++(int)
-    {
-        self_type retval = *this;
-        ptr++;
-        return retval;
-    }
-
-    /// Prefix increment
-    __host__ __device__ __forceinline__ self_type operator++()
-    {
-        ptr++;
-        return *this;
-    }
-
-    /// Indirection
-    __device__ __forceinline__ reference operator*() const
-    {
-        return ThreadLoad<MODIFIER>(ptr);
-    }
-
-    /// Addition
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type operator+(Distance n) const
-    {
-        self_type retval(ptr + n);
-        return retval;
-    }
-
-    /// Addition assignment
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator+=(Distance n)
-    {
-        ptr += n;
-        return *this;
-    }
-
-    /// Subtraction
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type operator-(Distance n) const
-    {
-        self_type retval(ptr - n);
-        return retval;
-    }
-
-    /// Subtraction assignment
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator-=(Distance n)
-    {
-        ptr -= n;
-        return *this;
-    }
-
-    /// Distance
-    __host__ __device__ __forceinline__ difference_type operator-(self_type other) const
-    {
-        return ptr - other.ptr;
-    }
-
-    /// Array subscript
-    template <typename Distance>
-    __device__ __forceinline__ reference operator[](Distance n) const
-    {
-        return ThreadLoad<MODIFIER>(ptr + n);
-    }
-
-    /// Structure dereference
-    __device__ __forceinline__ pointer operator->()
-    {
-        return &ThreadLoad<MODIFIER>(ptr);
-    }
-
-    /// Equal to
-    __host__ __device__ __forceinline__ bool operator==(const self_type& rhs)
-    {
-        return (ptr == rhs.ptr);
-    }
-
-    /// Not equal to
-    __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs)
-    {
-        return (ptr != rhs.ptr);
-    }
-
-    /// ostream operator
-    friend std::ostream& operator<<(std::ostream& os, const self_type& /*itr*/)
-    {
-        return os;
-    }
-};
-
-
-
-/** @} */       // end group UtilIterator
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/iterator/cache_modified_output_iterator.cuh b/pfsimulator/third_party/cub/iterator/cache_modified_output_iterator.cuh
deleted file mode 100644
index c3e3321d3..000000000
--- a/pfsimulator/third_party/cub/iterator/cache_modified_output_iterator.cuh
+++ /dev/null
@@ -1,254 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Random-access iterator types
- */
-
-#pragma once
-
-#include <iterator>
-#include <iostream>
-
-#include "../thread/thread_load.cuh"
-#include "../thread/thread_store.cuh"
-#include "../util_device.cuh"
-#include "../util_namespace.cuh"
-
-#if (THRUST_VERSION >= 100700)
-    // This iterator is compatible with Thrust API 1.7 and newer
-    #include <thrust/iterator/iterator_facade.h>
-    #include <thrust/iterator/iterator_traits.h>
-#endif // THRUST_VERSION
-
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \addtogroup UtilIterator
- * @{
- */
-
-
-/**
- * \brief A random-access output wrapper for storing array values using a PTX cache-modifier.
- *
- * \par Overview
- * - CacheModifiedOutputIterator is a random-access output iterator that wraps a native
- *   device pointer of type <tt>ValueType*</tt>. \p ValueType references are
- *   made by writing \p ValueType values through stores modified by \p MODIFIER.
- * - Can be used to store any data type to memory using PTX cache store modifiers (e.g., "STORE_WB",
- *   "STORE_CG", "STORE_CS", "STORE_WT", etc.).
- * - Can be constructed, manipulated, and exchanged within and between host and device
- *   functions, but can only be dereferenced within device functions.
- * - Compatible with Thrust API v1.7 or newer.
- *
- * \par Snippet
- * The code snippet below illustrates the use of \p CacheModifiedOutputIterator to
- * dereference a device array of doubles using the "wt" PTX load modifier
- * (i.e., write-through to system memory).
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/iterator/cache_modified_output_iterator.cuh>
- *
- * // Declare, allocate, and initialize a device array
- * double *d_out;              // e.g., [, , , , , , ]
- *
- * // Create an iterator wrapper
- * cub::CacheModifiedOutputIterator<cub::STORE_WT, double> itr(d_out);
- *
- * // Within device code:
- * itr[0]  = 8.0;
- * itr[1]  = 66.0;
- * itr[55] = 24.0;
- *
- * \endcode
- *
- * \par Usage Considerations
- * - Can only be dereferenced within device code
- *
- * \tparam CacheStoreModifier     The cub::CacheStoreModifier to use when accessing data
- * \tparam ValueType            The value type of this iterator
- * \tparam OffsetT              The difference type of this iterator (Default: \p ptrdiff_t)
- */
-template <
-    CacheStoreModifier  MODIFIER,
-    typename            ValueType,
-    typename            OffsetT = ptrdiff_t>
-class CacheModifiedOutputIterator
-{
-private:
-
-    // Proxy object
-    struct Reference
-    {
-        ValueType* ptr;
-
-        /// Constructor
-        __host__ __device__ __forceinline__ Reference(ValueType* ptr) : ptr(ptr) {}
-
-        /// Assignment
-        __device__ __forceinline__ ValueType operator =(ValueType val)
-        {
-            ThreadStore<MODIFIER>(ptr, val);
-            return val;
-        }
-    };
-
-public:
-
-    // Required iterator traits
-    typedef CacheModifiedOutputIterator         self_type;              ///< My own type
-    typedef OffsetT                             difference_type;        ///< Type to express the result of subtracting one iterator from another
-    typedef void                                value_type;             ///< The type of the element the iterator can point to
-    typedef void                                pointer;                ///< The type of a pointer to an element the iterator can point to
-    typedef Reference                           reference;              ///< The type of a reference to an element the iterator can point to
-
-#if (THRUST_VERSION >= 100700)
-    // Use Thrust's iterator categories so we can use these iterators in Thrust 1.7 (or newer) methods
-    typedef typename thrust::detail::iterator_facade_category<
-        thrust::device_system_tag,
-        thrust::random_access_traversal_tag,
-        value_type,
-        reference
-      >::type iterator_category;                                        ///< The iterator category
-#else
-    typedef std::random_access_iterator_tag     iterator_category;      ///< The iterator category
-#endif  // THRUST_VERSION
-
-private:
-
-    ValueType* ptr;
-
-public:
-
-    /// Constructor
-    template <typename QualifiedValueType>
-    __host__ __device__ __forceinline__ CacheModifiedOutputIterator(
-        QualifiedValueType* ptr)     ///< Native pointer to wrap
-    :
-        ptr(const_cast<typename RemoveQualifiers<QualifiedValueType>::Type *>(ptr))
-    {}
-
-    /// Postfix increment
-    __host__ __device__ __forceinline__ self_type operator++(int)
-    {
-        self_type retval = *this;
-        ptr++;
-        return retval;
-    }
-
-
-    /// Prefix increment
-    __host__ __device__ __forceinline__ self_type operator++()
-    {
-        ptr++;
-        return *this;
-    }
-
-    /// Indirection
-    __host__ __device__ __forceinline__ reference operator*() const
-    {
-        return Reference(ptr);
-    }
-
-    /// Addition
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type operator+(Distance n) const
-    {
-        self_type retval(ptr + n);
-        return retval;
-    }
-
-    /// Addition assignment
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator+=(Distance n)
-    {
-        ptr += n;
-        return *this;
-    }
-
-    /// Subtraction
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type operator-(Distance n) const
-    {
-        self_type retval(ptr - n);
-        return retval;
-    }
-
-    /// Subtraction assignment
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator-=(Distance n)
-    {
-        ptr -= n;
-        return *this;
-    }
-
-    /// Distance
-    __host__ __device__ __forceinline__ difference_type operator-(self_type other) const
-    {
-        return ptr - other.ptr;
-    }
-
-    /// Array subscript
-    template <typename Distance>
-    __host__ __device__ __forceinline__ reference operator[](Distance n) const
-    {
-        return Reference(ptr + n);
-    }
-
-    /// Equal to
-    __host__ __device__ __forceinline__ bool operator==(const self_type& rhs)
-    {
-        return (ptr == rhs.ptr);
-    }
-
-    /// Not equal to
-    __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs)
-    {
-        return (ptr != rhs.ptr);
-    }
-
-    /// ostream operator
-    friend std::ostream& operator<<(std::ostream& os, const self_type& itr)
-    {
-        return os;
-    }
-};
-
-
-/** @} */       // end group UtilIterator
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/iterator/constant_input_iterator.cuh b/pfsimulator/third_party/cub/iterator/constant_input_iterator.cuh
deleted file mode 100644
index 1e0a91044..000000000
--- a/pfsimulator/third_party/cub/iterator/constant_input_iterator.cuh
+++ /dev/null
@@ -1,235 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Random-access iterator types
- */
-
-#pragma once
-
-#include <iterator>
-#include <iostream>
-
-#include "../thread/thread_load.cuh"
-#include "../thread/thread_store.cuh"
-#include "../util_namespace.cuh"
-
-#if (THRUST_VERSION >= 100700)
-    // This iterator is compatible with Thrust API 1.7 and newer
-    #include <thrust/iterator/iterator_facade.h>
-    #include <thrust/iterator/iterator_traits.h>
-#endif // THRUST_VERSION
-
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \addtogroup UtilIterator
- * @{
- */
-
-
-/**
- * \brief A random-access input generator for dereferencing a sequence of homogeneous values
- *
- * \par Overview
- * - Read references to a ConstantInputIteratorTiterator always return the supplied constant
- *   of type \p ValueType.
- * - Can be used with any data type.
- * - Can be constructed, manipulated, dereferenced, and exchanged within and between host and device
- *   functions.
- * - Compatible with Thrust API v1.7 or newer.
- *
- * \par Snippet
- * The code snippet below illustrates the use of \p ConstantInputIteratorTto
- * dereference a sequence of homogeneous doubles.
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/iterator/constant_input_iterator.cuh>
- *
- * cub::ConstantInputIterator<double> itr(5.0);
- *
- * printf("%f\n", itr[0]);      // 5.0
- * printf("%f\n", itr[1]);      // 5.0
- * printf("%f\n", itr[2]);      // 5.0
- * printf("%f\n", itr[50]);     // 5.0
- *
- * \endcode
- *
- * \tparam ValueType            The value type of this iterator
- * \tparam OffsetT              The difference type of this iterator (Default: \p ptrdiff_t)
- */
-template <
-    typename ValueType,
-    typename OffsetT = ptrdiff_t>
-class ConstantInputIterator
-{
-public:
-
-    // Required iterator traits
-    typedef ConstantInputIterator               self_type;              ///< My own type
-    typedef OffsetT                             difference_type;        ///< Type to express the result of subtracting one iterator from another
-    typedef ValueType                           value_type;             ///< The type of the element the iterator can point to
-    typedef ValueType*                          pointer;                ///< The type of a pointer to an element the iterator can point to
-    typedef ValueType                           reference;              ///< The type of a reference to an element the iterator can point to
-
-#if (THRUST_VERSION >= 100700)
-    // Use Thrust's iterator categories so we can use these iterators in Thrust 1.7 (or newer) methods
-    typedef typename thrust::detail::iterator_facade_category<
-        thrust::any_system_tag,
-        thrust::random_access_traversal_tag,
-        value_type,
-        reference
-      >::type iterator_category;                                        ///< The iterator category
-#else
-    typedef std::random_access_iterator_tag     iterator_category;      ///< The iterator category
-#endif  // THRUST_VERSION
-
-private:
-
-    ValueType   val;
-    OffsetT     offset;
-#ifdef _WIN32
-    OffsetT     pad[CUB_MAX(1, (16 / sizeof(OffsetT) - 1))];        // Workaround for win32 parameter-passing bug (ulonglong2 argmin DeviceReduce)
-#endif
-
-public:
-
-    /// Constructor
-    __host__ __device__ __forceinline__ ConstantInputIterator(
-        ValueType   val,            ///< Starting value for the iterator instance to report
-        OffsetT     offset = 0)     ///< Base offset
-    :
-        val(val),
-        offset(offset)
-    {}
-
-    /// Postfix increment
-    __host__ __device__ __forceinline__ self_type operator++(int)
-    {
-        self_type retval = *this;
-        offset++;
-        return retval;
-    }
-
-    /// Prefix increment
-    __host__ __device__ __forceinline__ self_type operator++()
-    {
-        offset++;
-        return *this;
-    }
-
-    /// Indirection
-    __host__ __device__ __forceinline__ reference operator*() const
-    {
-        return val;
-    }
-
-    /// Addition
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type operator+(Distance n) const
-    {
-        self_type retval(val, offset + n);
-        return retval;
-    }
-
-    /// Addition assignment
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator+=(Distance n)
-    {
-        offset += n;
-        return *this;
-    }
-
-    /// Subtraction
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type operator-(Distance n) const
-    {
-        self_type retval(val, offset - n);
-        return retval;
-    }
-
-    /// Subtraction assignment
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator-=(Distance n)
-    {
-        offset -= n;
-        return *this;
-    }
-
-    /// Distance
-    __host__ __device__ __forceinline__ difference_type operator-(self_type other) const
-    {
-        return offset - other.offset;
-    }
-
-    /// Array subscript
-    template <typename Distance>
-    __host__ __device__ __forceinline__ reference operator[](Distance /*n*/) const
-    {
-        return val;
-    }
-
-    /// Structure dereference
-    __host__ __device__ __forceinline__ pointer operator->()
-    {
-        return &val;
-    }
-
-    /// Equal to
-    __host__ __device__ __forceinline__ bool operator==(const self_type& rhs)
-    {
-        return (offset == rhs.offset) && ((val == rhs.val));
-    }
-
-    /// Not equal to
-    __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs)
-    {
-        return (offset != rhs.offset) || (val!= rhs.val);
-    }
-
-    /// ostream operator
-    friend std::ostream& operator<<(std::ostream& os, const self_type& itr)
-    {
-        os << "[" << itr.val << "," << itr.offset << "]";
-        return os;
-    }
-
-};
-
-
-/** @} */       // end group UtilIterator
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/iterator/counting_input_iterator.cuh b/pfsimulator/third_party/cub/iterator/counting_input_iterator.cuh
deleted file mode 100644
index 7f49348d6..000000000
--- a/pfsimulator/third_party/cub/iterator/counting_input_iterator.cuh
+++ /dev/null
@@ -1,228 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Random-access iterator types
- */
-
-#pragma once
-
-#include <iterator>
-#include <iostream>
-
-#include "../thread/thread_load.cuh"
-#include "../thread/thread_store.cuh"
-#include "../util_device.cuh"
-#include "../util_namespace.cuh"
-
-#if (THRUST_VERSION >= 100700)
-    // This iterator is compatible with Thrust API 1.7 and newer
-    #include <thrust/iterator/iterator_facade.h>
-    #include <thrust/iterator/iterator_traits.h>
-#endif // THRUST_VERSION
-
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \addtogroup UtilIterator
- * @{
- */
-
-/**
- * \brief A random-access input generator for dereferencing a sequence of incrementing integer values.
- *
- * \par Overview
- * - After initializing a CountingInputIteratorTto a certain integer \p base, read references
- *   at \p offset will return the value \p base + \p offset.
- * - Can be constructed, manipulated, dereferenced, and exchanged within and between host and device
- *   functions.
- * - Compatible with Thrust API v1.7 or newer.
- *
- * \par Snippet
- * The code snippet below illustrates the use of \p CountingInputIteratorTto
- * dereference a sequence of incrementing integers.
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/iterator/counting_input_iterator.cuh>
- *
- * cub::CountingInputIterator<int> itr(5);
- *
- * printf("%d\n", itr[0]);      // 5
- * printf("%d\n", itr[1]);      // 6
- * printf("%d\n", itr[2]);      // 7
- * printf("%d\n", itr[50]);     // 55
- *
- * \endcode
- *
- * \tparam ValueType            The value type of this iterator
- * \tparam OffsetT              The difference type of this iterator (Default: \p ptrdiff_t)
- */
-template <
-    typename ValueType,
-    typename OffsetT = ptrdiff_t>
-class CountingInputIterator
-{
-public:
-
-    // Required iterator traits
-    typedef CountingInputIterator               self_type;              ///< My own type
-    typedef OffsetT                             difference_type;        ///< Type to express the result of subtracting one iterator from another
-    typedef ValueType                           value_type;             ///< The type of the element the iterator can point to
-    typedef ValueType*                          pointer;                ///< The type of a pointer to an element the iterator can point to
-    typedef ValueType                           reference;              ///< The type of a reference to an element the iterator can point to
-
-#if (THRUST_VERSION >= 100700)
-    // Use Thrust's iterator categories so we can use these iterators in Thrust 1.7 (or newer) methods
-    typedef typename thrust::detail::iterator_facade_category<
-        thrust::any_system_tag,
-        thrust::random_access_traversal_tag,
-        value_type,
-        reference
-      >::type iterator_category;                                        ///< The iterator category
-#else
-    typedef std::random_access_iterator_tag     iterator_category;      ///< The iterator category
-#endif  // THRUST_VERSION
-
-private:
-
-    ValueType val;
-
-public:
-
-    /// Constructor
-    __host__ __device__ __forceinline__ CountingInputIterator(
-        const ValueType &val)          ///< Starting value for the iterator instance to report
-    :
-        val(val)
-    {}
-
-    /// Postfix increment
-    __host__ __device__ __forceinline__ self_type operator++(int)
-    {
-        self_type retval = *this;
-        val++;
-        return retval;
-    }
-
-    /// Prefix increment
-    __host__ __device__ __forceinline__ self_type operator++()
-    {
-        val++;
-        return *this;
-    }
-
-    /// Indirection
-    __host__ __device__ __forceinline__ reference operator*() const
-    {
-        return val;
-    }
-
-    /// Addition
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type operator+(Distance n) const
-    {
-        self_type retval(val + (ValueType) n);
-        return retval;
-    }
-
-    /// Addition assignment
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator+=(Distance n)
-    {
-        val += (ValueType) n;
-        return *this;
-    }
-
-    /// Subtraction
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type operator-(Distance n) const
-    {
-        self_type retval(val - (ValueType) n);
-        return retval;
-    }
-
-    /// Subtraction assignment
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator-=(Distance n)
-    {
-        val -= n;
-        return *this;
-    }
-
-    /// Distance
-    __host__ __device__ __forceinline__ difference_type operator-(self_type other) const
-    {
-        return (difference_type) (val - other.val);
-    }
-
-    /// Array subscript
-    template <typename Distance>
-    __host__ __device__ __forceinline__ reference operator[](Distance n) const
-    {
-        return val + (ValueType) n;
-    }
-
-    /// Structure dereference
-    __host__ __device__ __forceinline__ pointer operator->()
-    {
-        return &val;
-    }
-
-    /// Equal to
-    __host__ __device__ __forceinline__ bool operator==(const self_type& rhs)
-    {
-        return (val == rhs.val);
-    }
-
-    /// Not equal to
-    __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs)
-    {
-        return (val != rhs.val);
-    }
-
-    /// ostream operator
-    friend std::ostream& operator<<(std::ostream& os, const self_type& itr)
-    {
-        os << "[" << itr.val << "]";
-        return os;
-    }
-
-};
-
-
-
-/** @} */       // end group UtilIterator
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/iterator/discard_output_iterator.cuh b/pfsimulator/third_party/cub/iterator/discard_output_iterator.cuh
deleted file mode 100644
index 28473e5f2..000000000
--- a/pfsimulator/third_party/cub/iterator/discard_output_iterator.cuh
+++ /dev/null
@@ -1,220 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Random-access iterator types
- */
-
-#pragma once
-
-#include <iterator>
-#include <iostream>
-
-#include "../util_namespace.cuh"
-#include "../util_macro.cuh"
-
-#if (THRUST_VERSION >= 100700)
-    // This iterator is compatible with Thrust API 1.7 and newer
-    #include <thrust/iterator/iterator_facade.h>
-    #include <thrust/iterator/iterator_traits.h>
-#endif // THRUST_VERSION
-
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \addtogroup UtilIterator
- * @{
- */
-
-
-/**
- * \brief A discard iterator
- */
-template <typename OffsetT = ptrdiff_t>
-class DiscardOutputIterator
-{
-public:
-
-    // Required iterator traits
-    typedef DiscardOutputIterator   self_type;              ///< My own type
-    typedef OffsetT                 difference_type;        ///< Type to express the result of subtracting one iterator from another
-    typedef void                    value_type;             ///< The type of the element the iterator can point to
-    typedef void                    pointer;                ///< The type of a pointer to an element the iterator can point to
-    typedef void                    reference;              ///< The type of a reference to an element the iterator can point to
-
-#if (THRUST_VERSION >= 100700)
-    // Use Thrust's iterator categories so we can use these iterators in Thrust 1.7 (or newer) methods
-    typedef typename thrust::detail::iterator_facade_category<
-        thrust::any_system_tag,
-        thrust::random_access_traversal_tag,
-        value_type,
-        reference
-      >::type iterator_category;                                        ///< The iterator category
-#else
-    typedef std::random_access_iterator_tag     iterator_category;      ///< The iterator category
-#endif  // THRUST_VERSION
-
-private:
-
-    OffsetT offset;
-
-#if defined(_WIN32) || !defined(_WIN64)
-    // Workaround for win32 parameter-passing bug (ulonglong2 argmin DeviceReduce)
-    OffsetT pad[CUB_MAX(1, (16 / sizeof(OffsetT) - 1))];
-#endif
-
-public:
-
-    /// Constructor
-    __host__ __device__ __forceinline__ DiscardOutputIterator(
-        OffsetT offset = 0)     ///< Base offset
-    :
-        offset(offset)
-    {}
-
-    /// Postfix increment
-    __host__ __device__ __forceinline__ self_type operator++(int)
-    {
-        self_type retval = *this;
-        offset++;
-        return retval;
-    }
-
-    /// Prefix increment
-    __host__ __device__ __forceinline__ self_type operator++()
-    {
-        offset++;
-        return *this;
-    }
-
-    /// Indirection
-    __host__ __device__ __forceinline__ self_type& operator*()
-    {
-        // return self reference, which can be assigned to anything
-        return *this;
-    }
-
-    /// Addition
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type operator+(Distance n) const
-    {
-        self_type retval(offset + n);
-        return retval;
-    }
-
-    /// Addition assignment
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator+=(Distance n)
-    {
-        offset += n;
-        return *this;
-    }
-
-    /// Subtraction
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type operator-(Distance n) const
-    {
-        self_type retval(offset - n);
-        return retval;
-    }
-
-    /// Subtraction assignment
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator-=(Distance n)
-    {
-        offset -= n;
-        return *this;
-    }
-
-    /// Distance
-    __host__ __device__ __forceinline__ difference_type operator-(self_type other) const
-    {
-        return offset - other.offset;
-    }
-
-    /// Array subscript
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator[](Distance n)
-    {
-        // return self reference, which can be assigned to anything
-        return *this;
-    }
-
-    /// Structure dereference
-    __host__ __device__ __forceinline__ pointer operator->()
-    {
-        return;
-    }
-
-    /// Assignment to self (no-op)
-    __host__ __device__ __forceinline__ void operator=(self_type const& other)
-    {
-        offset = other.offset;
-    }
-
-    /// Assignment to anything else (no-op)
-    template<typename T>
-    __host__ __device__ __forceinline__ void operator=(T const&)
-    {}
-
-    /// Cast to void* operator
-    __host__ __device__ __forceinline__ operator void*() const { return NULL; }
-
-    /// Equal to
-    __host__ __device__ __forceinline__ bool operator==(const self_type& rhs)
-    {
-        return (offset == rhs.offset);
-    }
-
-    /// Not equal to
-    __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs)
-    {
-        return (offset != rhs.offset);
-    }
-
-    /// ostream operator
-    friend std::ostream& operator<<(std::ostream& os, const self_type& itr)
-    {
-        os << "[" << itr.offset << "]";
-        return os;
-    }
-
-};
-
-
-/** @} */       // end group UtilIterator
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/iterator/tex_obj_input_iterator.cuh b/pfsimulator/third_party/cub/iterator/tex_obj_input_iterator.cuh
deleted file mode 100644
index b99103ec5..000000000
--- a/pfsimulator/third_party/cub/iterator/tex_obj_input_iterator.cuh
+++ /dev/null
@@ -1,310 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Random-access iterator types
- */
-
-#pragma once
-
-#include <iterator>
-#include <iostream>
-
-#include "../thread/thread_load.cuh"
-#include "../thread/thread_store.cuh"
-#include "../util_device.cuh"
-#include "../util_debug.cuh"
-#include "../util_namespace.cuh"
-
-#if (THRUST_VERSION >= 100700)
-    // This iterator is compatible with Thrust API 1.7 and newer
-    #include <thrust/iterator/iterator_facade.h>
-    #include <thrust/iterator/iterator_traits.h>
-#endif // THRUST_VERSION
-
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \addtogroup UtilIterator
- * @{
- */
-
-
-
-/**
- * \brief A random-access input wrapper for dereferencing array values through texture cache.  Uses newer Kepler-style texture objects.
- *
- * \par Overview
- * - TexObjInputIteratorTwraps a native device pointer of type <tt>ValueType*</tt>. References
- *   to elements are to be loaded through texture cache.
- * - Can be used to load any data type from memory through texture cache.
- * - Can be manipulated and exchanged within and between host and device
- *   functions, can only be constructed within host functions, and can only be
- *   dereferenced within device functions.
- * - With regard to nested/dynamic parallelism, TexObjInputIteratorTiterators may only be
- *   created by the host thread, but can be used by any descendant kernel.
- * - Compatible with Thrust API v1.7 or newer.
- *
- * \par Snippet
- * The code snippet below illustrates the use of \p TexRefInputIteratorTto
- * dereference a device array of doubles through texture cache.
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/iterator/tex_obj_input_iterator.cuh>
- *
- * // Declare, allocate, and initialize a device array
- * int num_items;   // e.g., 7
- * double *d_in;    // e.g., [8.0, 6.0, 7.0, 5.0, 3.0, 0.0, 9.0]
- *
- * // Create an iterator wrapper
- * cub::TexObjInputIterator<double> itr;
- * itr.BindTexture(d_in, sizeof(double) * num_items);
- * ...
- *
- * // Within device code:
- * printf("%f\n", itr[0]);      // 8.0
- * printf("%f\n", itr[1]);      // 6.0
- * printf("%f\n", itr[6]);      // 9.0
- *
- * ...
- * itr.UnbindTexture();
- *
- * \endcode
- *
- * \tparam T                    The value type of this iterator
- * \tparam OffsetT              The difference type of this iterator (Default: \p ptrdiff_t)
- */
-template <
-    typename    T,
-    typename    OffsetT = ptrdiff_t>
-class TexObjInputIterator
-{
-public:
-
-    // Required iterator traits
-    typedef TexObjInputIterator                 self_type;              ///< My own type
-    typedef OffsetT                             difference_type;        ///< Type to express the result of subtracting one iterator from another
-    typedef T                                   value_type;             ///< The type of the element the iterator can point to
-    typedef T*                                  pointer;                ///< The type of a pointer to an element the iterator can point to
-    typedef T                                   reference;              ///< The type of a reference to an element the iterator can point to
-
-#if (THRUST_VERSION >= 100700)
-    // Use Thrust's iterator categories so we can use these iterators in Thrust 1.7 (or newer) methods
-    typedef typename thrust::detail::iterator_facade_category<
-        thrust::device_system_tag,
-        thrust::random_access_traversal_tag,
-        value_type,
-        reference
-      >::type iterator_category;                                        ///< The iterator category
-#else
-    typedef std::random_access_iterator_tag     iterator_category;      ///< The iterator category
-#endif  // THRUST_VERSION
-
-private:
-
-    // Largest texture word we can use in device
-    typedef typename UnitWord<T>::TextureWord TextureWord;
-
-    // Number of texture words per T
-    enum {
-        TEXTURE_MULTIPLE = sizeof(T) / sizeof(TextureWord)
-    };
-
-private:
-
-    T*                  ptr;
-    difference_type     tex_offset;
-    cudaTextureObject_t tex_obj;
-
-public:
-
-    /// Constructor
-    __host__ __device__ __forceinline__ TexObjInputIterator()
-    :
-        ptr(NULL),
-        tex_offset(0),
-        tex_obj(0)
-    {}
-
-    /// Use this iterator to bind \p ptr with a texture reference
-    template <typename QualifiedT>
-    cudaError_t BindTexture(
-        QualifiedT      *ptr,               ///< Native pointer to wrap that is aligned to cudaDeviceProp::textureAlignment
-        size_t          bytes = size_t(-1),         ///< Number of bytes in the range
-        size_t          tex_offset = 0)     ///< OffsetT (in items) from \p ptr denoting the position of the iterator
-    {
-        this->ptr = const_cast<typename RemoveQualifiers<QualifiedT>::Type *>(ptr);
-        this->tex_offset = tex_offset;
-
-        cudaChannelFormatDesc   channel_desc = cudaCreateChannelDesc<TextureWord>();
-        cudaResourceDesc        res_desc;
-        cudaTextureDesc         tex_desc;
-        memset(&res_desc, 0, sizeof(cudaResourceDesc));
-        memset(&tex_desc, 0, sizeof(cudaTextureDesc));
-        res_desc.resType                = cudaResourceTypeLinear;
-        res_desc.res.linear.devPtr      = this->ptr;
-        res_desc.res.linear.desc        = channel_desc;
-        res_desc.res.linear.sizeInBytes = bytes;
-        tex_desc.readMode               = cudaReadModeElementType;
-        return cudaCreateTextureObject(&tex_obj, &res_desc, &tex_desc, NULL);
-    }
-
-    /// Unbind this iterator from its texture reference
-    cudaError_t UnbindTexture()
-    {
-        return cudaDestroyTextureObject(tex_obj);
-    }
-
-    /// Postfix increment
-    __host__ __device__ __forceinline__ self_type operator++(int)
-    {
-        self_type retval = *this;
-        tex_offset++;
-        return retval;
-    }
-
-    /// Prefix increment
-    __host__ __device__ __forceinline__ self_type operator++()
-    {
-        tex_offset++;
-        return *this;
-    }
-
-    /// Indirection
-    __host__ __device__ __forceinline__ reference operator*() const
-    {
-#if (CUB_PTX_ARCH == 0)
-        // Simply dereference the pointer on the host
-        return ptr[tex_offset];
-#else
-        // Move array of uninitialized words, then alias and assign to return value
-        TextureWord words[TEXTURE_MULTIPLE];
-
-        #pragma unroll
-        for (int i = 0; i < TEXTURE_MULTIPLE; ++i)
-        {
-            words[i] = tex1Dfetch<TextureWord>(
-                tex_obj,
-                (tex_offset * TEXTURE_MULTIPLE) + i);
-        }
-
-        // Load from words
-        return *reinterpret_cast<T*>(words);
-#endif
-    }
-
-    /// Addition
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type operator+(Distance n) const
-    {
-        self_type retval;
-        retval.ptr          = ptr;
-        retval.tex_obj      = tex_obj;
-        retval.tex_offset   = tex_offset + n;
-        return retval;
-    }
-
-    /// Addition assignment
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator+=(Distance n)
-    {
-        tex_offset += n;
-        return *this;
-    }
-
-    /// Subtraction
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type operator-(Distance n) const
-    {
-        self_type retval;
-        retval.ptr          = ptr;
-        retval.tex_obj      = tex_obj;
-        retval.tex_offset   = tex_offset - n;
-        return retval;
-    }
-
-    /// Subtraction assignment
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator-=(Distance n)
-    {
-        tex_offset -= n;
-        return *this;
-    }
-
-    /// Distance
-    __host__ __device__ __forceinline__ difference_type operator-(self_type other) const
-    {
-        return tex_offset - other.tex_offset;
-    }
-
-    /// Array subscript
-    template <typename Distance>
-    __host__ __device__ __forceinline__ reference operator[](Distance n) const
-    {
-        self_type offset = (*this) + n;
-        return *offset;
-    }
-
-    /// Structure dereference
-    __host__ __device__ __forceinline__ pointer operator->()
-    {
-        return &(*(*this));
-    }
-
-    /// Equal to
-    __host__ __device__ __forceinline__ bool operator==(const self_type& rhs)
-    {
-        return ((ptr == rhs.ptr) && (tex_offset == rhs.tex_offset) && (tex_obj == rhs.tex_obj));
-    }
-
-    /// Not equal to
-    __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs)
-    {
-        return ((ptr != rhs.ptr) || (tex_offset != rhs.tex_offset) || (tex_obj != rhs.tex_obj));
-    }
-
-    /// ostream operator
-    friend std::ostream& operator<<(std::ostream& os, const self_type& itr)
-    {
-        return os;
-    }
-
-};
-
-
-
-/** @} */       // end group UtilIterator
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/iterator/tex_ref_input_iterator.cuh b/pfsimulator/third_party/cub/iterator/tex_ref_input_iterator.cuh
deleted file mode 100644
index 95d0ffbc9..000000000
--- a/pfsimulator/third_party/cub/iterator/tex_ref_input_iterator.cuh
+++ /dev/null
@@ -1,374 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Random-access iterator types
- */
-
-#pragma once
-
-#include <iterator>
-#include <iostream>
-
-#include "../thread/thread_load.cuh"
-#include "../thread/thread_store.cuh"
-#include "../util_device.cuh"
-#include "../util_debug.cuh"
-#include "../util_namespace.cuh"
-
-#if (CUDA_VERSION >= 5050) || defined(DOXYGEN_ACTIVE)  // This iterator is compatible with CUDA 5.5 and newer
-
-#if (THRUST_VERSION >= 100700)    // This iterator is compatible with Thrust API 1.7 and newer
-    #include <thrust/iterator/iterator_facade.h>
-    #include <thrust/iterator/iterator_traits.h>
-#endif // THRUST_VERSION
-
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/******************************************************************************
- * Static file-scope Tesla/Fermi-style texture references
- *****************************************************************************/
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-// Anonymous namespace
-namespace {
-
-/// Global texture reference specialized by type
-template <typename T>
-struct IteratorTexRef
-{
-    /// And by unique ID
-    template <int UNIQUE_ID>
-    struct TexId
-    {
-        // Largest texture word we can use in device
-        typedef typename UnitWord<T>::DeviceWord DeviceWord;
-        typedef typename UnitWord<T>::TextureWord TextureWord;
-
-        // Number of texture words per T
-        enum {
-            DEVICE_MULTIPLE = sizeof(T) / sizeof(DeviceWord),
-            TEXTURE_MULTIPLE = sizeof(T) / sizeof(TextureWord)
-        };
-
-        // Texture reference type
-        typedef texture<TextureWord> TexRef;
-
-        // Texture reference
-        static TexRef ref;
-
-        /// Bind texture
-        static cudaError_t BindTexture(void *d_in, size_t &offset)
-        {
-            if (d_in)
-            {
-                cudaChannelFormatDesc tex_desc = cudaCreateChannelDesc<TextureWord>();
-                ref.channelDesc = tex_desc;
-                return (CubDebug(cudaBindTexture(&offset, ref, d_in)));
-            }
-
-            return cudaSuccess;
-        }
-
-        /// Unbind texture
-        static cudaError_t UnbindTexture()
-        {
-            return CubDebug(cudaUnbindTexture(ref));
-        }
-
-        /// Fetch element
-        template <typename Distance>
-        static __device__ __forceinline__ T Fetch(Distance tex_offset)
-        {
-            DeviceWord temp[DEVICE_MULTIPLE];
-            TextureWord *words = reinterpret_cast<TextureWord*>(temp);
-
-            #pragma unroll
-            for (int i = 0; i < TEXTURE_MULTIPLE; ++i)
-            {
-                words[i] = tex1Dfetch(ref, (tex_offset * TEXTURE_MULTIPLE) + i);
-            }
-
-            return reinterpret_cast<T&>(temp);
-        }
-    };
-};
-
-// Texture reference definitions
-template <typename  T>
-template <int       UNIQUE_ID>
-typename IteratorTexRef<T>::template TexId<UNIQUE_ID>::TexRef IteratorTexRef<T>::template TexId<UNIQUE_ID>::ref = 0;
-
-
-} // Anonymous namespace
-
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-
-/**
- * \addtogroup UtilIterator
- * @{
- */
-
-
-
-/**
- * \brief A random-access input wrapper for dereferencing array values through texture cache.  Uses older Tesla/Fermi-style texture references.
- *
- * \par Overview
- * - TexRefInputIteratorTwraps a native device pointer of type <tt>ValueType*</tt>. References
- *   to elements are to be loaded through texture cache.
- * - Can be used to load any data type from memory through texture cache.
- * - Can be manipulated and exchanged within and between host and device
- *   functions, can only be constructed within host functions, and can only be
- *   dereferenced within device functions.
- * - The \p UNIQUE_ID template parameter is used to statically name the underlying texture
- *   reference.  Only one TexRefInputIteratorTinstance can be bound at any given time for a
- *   specific combination of (1) data type \p T, (2) \p UNIQUE_ID, (3) host
- *   thread, and (4) compilation .o unit.
- * - With regard to nested/dynamic parallelism, TexRefInputIteratorTiterators may only be
- *   created by the host thread and used by a top-level kernel (i.e. the one which is launched
- *   from the host).
- * - Compatible with Thrust API v1.7 or newer.
- * - Compatible with CUDA toolkit v5.5 or newer.
- *
- * \par Snippet
- * The code snippet below illustrates the use of \p TexRefInputIteratorTto
- * dereference a device array of doubles through texture cache.
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/iterator/tex_ref_input_iterator.cuh>
- *
- * // Declare, allocate, and initialize a device array
- * int num_items;   // e.g., 7
- * double *d_in;    // e.g., [8.0, 6.0, 7.0, 5.0, 3.0, 0.0, 9.0]
- *
- * // Create an iterator wrapper
- * cub::TexRefInputIterator<double, __LINE__> itr;
- * itr.BindTexture(d_in, sizeof(double) * num_items);
- * ...
- *
- * // Within device code:
- * printf("%f\n", itr[0]);      // 8.0
- * printf("%f\n", itr[1]);      // 6.0
- * printf("%f\n", itr[6]);      // 9.0
- *
- * ...
- * itr.UnbindTexture();
- *
- * \endcode
- *
- * \tparam T                    The value type of this iterator
- * \tparam UNIQUE_ID            A globally-unique identifier (within the compilation unit) to name the underlying texture reference
- * \tparam OffsetT              The difference type of this iterator (Default: \p ptrdiff_t)
- */
-template <
-    typename    T,
-    int         UNIQUE_ID,
-    typename    OffsetT = ptrdiff_t>
-class TexRefInputIterator
-{
-public:
-
-    // Required iterator traits
-    typedef TexRefInputIterator                 self_type;              ///< My own type
-    typedef OffsetT                             difference_type;        ///< Type to express the result of subtracting one iterator from another
-    typedef T                                   value_type;             ///< The type of the element the iterator can point to
-    typedef T*                                  pointer;                ///< The type of a pointer to an element the iterator can point to
-    typedef T                                   reference;              ///< The type of a reference to an element the iterator can point to
-
-#if (THRUST_VERSION >= 100700)
-    // Use Thrust's iterator categories so we can use these iterators in Thrust 1.7 (or newer) methods
-    typedef typename thrust::detail::iterator_facade_category<
-        thrust::device_system_tag,
-        thrust::random_access_traversal_tag,
-        value_type,
-        reference
-      >::type iterator_category;                                        ///< The iterator category
-#else
-    typedef std::random_access_iterator_tag     iterator_category;      ///< The iterator category
-#endif  // THRUST_VERSION
-
-private:
-
-    T*              ptr;
-    difference_type tex_offset;
-
-    // Texture reference wrapper (old Tesla/Fermi-style textures)
-    typedef typename IteratorTexRef<T>::template TexId<UNIQUE_ID> TexId;
-
-public:
-/*
-    /// Constructor
-    __host__ __device__ __forceinline__ TexRefInputIterator()
-    :
-        ptr(NULL),
-        tex_offset(0)
-    {}
-*/
-    /// Use this iterator to bind \p ptr with a texture reference
-    template <typename QualifiedT>
-    cudaError_t BindTexture(
-        QualifiedT      *ptr,                   ///< Native pointer to wrap that is aligned to cudaDeviceProp::textureAlignment
-        size_t          bytes = size_t(-1),     ///< Number of bytes in the range
-        size_t          tex_offset = 0)         ///< OffsetT (in items) from \p ptr denoting the position of the iterator
-    {
-        this->ptr = const_cast<typename RemoveQualifiers<QualifiedT>::Type *>(ptr);
-        size_t offset;
-        cudaError_t retval = TexId::BindTexture(this->ptr + tex_offset, offset);
-        this->tex_offset = (difference_type) (offset / sizeof(QualifiedT));
-        return retval;
-    }
-
-    /// Unbind this iterator from its texture reference
-    cudaError_t UnbindTexture()
-    {
-        return TexId::UnbindTexture();
-    }
-
-    /// Postfix increment
-    __host__ __device__ __forceinline__ self_type operator++(int)
-    {
-        self_type retval = *this;
-        tex_offset++;
-        return retval;
-    }
-
-    /// Prefix increment
-    __host__ __device__ __forceinline__ self_type operator++()
-    {
-        tex_offset++;
-        return *this;
-    }
-
-    /// Indirection
-    __host__ __device__ __forceinline__ reference operator*() const
-    {
-#if (CUB_PTX_ARCH == 0)
-        // Simply dereference the pointer on the host
-        return ptr[tex_offset];
-#else
-        // Use the texture reference
-        return TexId::Fetch(tex_offset);
-#endif
-    }
-
-    /// Addition
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type operator+(Distance n) const
-    {
-        self_type retval;
-        retval.ptr = ptr;
-        retval.tex_offset = tex_offset + n;
-        return retval;
-    }
-
-    /// Addition assignment
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator+=(Distance n)
-    {
-        tex_offset += n;
-        return *this;
-    }
-
-    /// Subtraction
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type operator-(Distance n) const
-    {
-        self_type retval;
-        retval.ptr = ptr;
-        retval.tex_offset = tex_offset - n;
-        return retval;
-    }
-
-    /// Subtraction assignment
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator-=(Distance n)
-    {
-        tex_offset -= n;
-        return *this;
-    }
-
-    /// Distance
-    __host__ __device__ __forceinline__ difference_type operator-(self_type other) const
-    {
-        return tex_offset - other.tex_offset;
-    }
-
-    /// Array subscript
-    template <typename Distance>
-    __host__ __device__ __forceinline__ reference operator[](Distance n) const
-    {
-        self_type offset = (*this) + n;
-        return *offset;
-    }
-
-    /// Structure dereference
-    __host__ __device__ __forceinline__ pointer operator->()
-    {
-        return &(*(*this));
-    }
-
-    /// Equal to
-    __host__ __device__ __forceinline__ bool operator==(const self_type& rhs)
-    {
-        return ((ptr == rhs.ptr) && (tex_offset == rhs.tex_offset));
-    }
-
-    /// Not equal to
-    __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs)
-    {
-        return ((ptr != rhs.ptr) || (tex_offset != rhs.tex_offset));
-    }
-
-    /// ostream operator
-    friend std::ostream& operator<<(std::ostream& os, const self_type& itr)
-    {
-        return os;
-    }
-
-};
-
-
-
-/** @} */       // end group UtilIterator
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
-
-#endif // CUDA_VERSION
diff --git a/pfsimulator/third_party/cub/iterator/transform_input_iterator.cuh b/pfsimulator/third_party/cub/iterator/transform_input_iterator.cuh
deleted file mode 100644
index dad1f5004..000000000
--- a/pfsimulator/third_party/cub/iterator/transform_input_iterator.cuh
+++ /dev/null
@@ -1,252 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Random-access iterator types
- */
-
-#pragma once
-
-#include <iterator>
-#include <iostream>
-
-#include "../thread/thread_load.cuh"
-#include "../thread/thread_store.cuh"
-#include "../util_device.cuh"
-#include "../util_namespace.cuh"
-
-#if (THRUST_VERSION >= 100700)
-    // This iterator is compatible with Thrust API 1.7 and newer
-    #include <thrust/iterator/iterator_facade.h>
-    #include <thrust/iterator/iterator_traits.h>
-#endif // THRUST_VERSION
-
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \addtogroup UtilIterator
- * @{
- */
-
-
-/**
- * \brief A random-access input wrapper for transforming dereferenced values.
- *
- * \par Overview
- * - TransformInputIteratorTwraps a unary conversion functor of type \p
- *   ConversionOp and a random-access input iterator of type <tt>InputIteratorT</tt>,
- *   using the former to produce references of type \p ValueType from the latter.
- * - Can be used with any data type.
- * - Can be constructed, manipulated, and exchanged within and between host and device
- *   functions.  Wrapped host memory can only be dereferenced on the host, and wrapped
- *   device memory can only be dereferenced on the device.
- * - Compatible with Thrust API v1.7 or newer.
- *
- * \par Snippet
- * The code snippet below illustrates the use of \p TransformInputIteratorTto
- * dereference an array of integers, tripling the values and converting them to doubles.
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/iterator/transform_input_iterator.cuh>
- *
- * // Functor for tripling integer values and converting to doubles
- * struct TripleDoubler
- * {
- *     __host__ __device__ __forceinline__
- *     double operator()(const int &a) const {
- *         return double(a * 3);
- *     }
- * };
- *
- * // Declare, allocate, and initialize a device array
- * int *d_in;                   // e.g., [8, 6, 7, 5, 3, 0, 9]
- * TripleDoubler conversion_op;
- *
- * // Create an iterator wrapper
- * cub::TransformInputIterator<double, TripleDoubler, int*> itr(d_in, conversion_op);
- *
- * // Within device code:
- * printf("%f\n", itr[0]);  // 24.0
- * printf("%f\n", itr[1]);  // 18.0
- * printf("%f\n", itr[6]);  // 27.0
- *
- * \endcode
- *
- * \tparam ValueType            The value type of this iterator
- * \tparam ConversionOp         Unary functor type for mapping objects of type \p InputType to type \p ValueType.  Must have member <tt>ValueType operator()(const InputType &datum)</tt>.
- * \tparam InputIteratorT       The type of the wrapped input iterator
- * \tparam OffsetT              The difference type of this iterator (Default: \p ptrdiff_t)
- *
- */
-template <
-    typename ValueType,
-    typename ConversionOp,
-    typename InputIteratorT,
-    typename OffsetT = ptrdiff_t>
-class TransformInputIterator
-{
-public:
-
-    // Required iterator traits
-    typedef TransformInputIterator              self_type;              ///< My own type
-    typedef OffsetT                             difference_type;        ///< Type to express the result of subtracting one iterator from another
-    typedef ValueType                           value_type;             ///< The type of the element the iterator can point to
-    typedef ValueType*                          pointer;                ///< The type of a pointer to an element the iterator can point to
-    typedef ValueType                           reference;              ///< The type of a reference to an element the iterator can point to
-
-#if (THRUST_VERSION >= 100700)
-    // Use Thrust's iterator categories so we can use these iterators in Thrust 1.7 (or newer) methods
-    typedef typename thrust::detail::iterator_facade_category<
-        thrust::any_system_tag,
-        thrust::random_access_traversal_tag,
-        value_type,
-        reference
-      >::type iterator_category;                                        ///< The iterator category
-#else
-    typedef std::random_access_iterator_tag     iterator_category;      ///< The iterator category
-#endif  // THRUST_VERSION
-
-private:
-
-    ConversionOp    conversion_op;
-    InputIteratorT  input_itr;
-
-public:
-
-    /// Constructor
-    __host__ __device__ __forceinline__ TransformInputIterator(
-        InputIteratorT      input_itr,          ///< Input iterator to wrap
-        ConversionOp        conversion_op)      ///< Conversion functor to wrap
-    :
-        conversion_op(conversion_op),
-        input_itr(input_itr)
-    {}
-
-    /// Postfix increment
-    __host__ __device__ __forceinline__ self_type operator++(int)
-    {
-        self_type retval = *this;
-        input_itr++;
-        return retval;
-    }
-
-    /// Prefix increment
-    __host__ __device__ __forceinline__ self_type operator++()
-    {
-        input_itr++;
-        return *this;
-    }
-
-    /// Indirection
-    __host__ __device__ __forceinline__ reference operator*() const
-    {
-        return conversion_op(*input_itr);
-    }
-
-    /// Addition
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type operator+(Distance n) const
-    {
-        self_type retval(input_itr + n, conversion_op);
-        return retval;
-    }
-
-    /// Addition assignment
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator+=(Distance n)
-    {
-        input_itr += n;
-        return *this;
-    }
-
-    /// Subtraction
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type operator-(Distance n) const
-    {
-        self_type retval(input_itr - n, conversion_op);
-        return retval;
-    }
-
-    /// Subtraction assignment
-    template <typename Distance>
-    __host__ __device__ __forceinline__ self_type& operator-=(Distance n)
-    {
-        input_itr -= n;
-        return *this;
-    }
-
-    /// Distance
-    __host__ __device__ __forceinline__ difference_type operator-(self_type other) const
-    {
-        return input_itr - other.input_itr;
-    }
-
-    /// Array subscript
-    template <typename Distance>
-    __host__ __device__ __forceinline__ reference operator[](Distance n) const
-    {
-        return conversion_op(input_itr[n]);
-    }
-
-    /// Structure dereference
-    __host__ __device__ __forceinline__ pointer operator->()
-    {
-        return &conversion_op(*input_itr);
-    }
-
-    /// Equal to
-    __host__ __device__ __forceinline__ bool operator==(const self_type& rhs)
-    {
-        return (input_itr == rhs.input_itr);
-    }
-
-    /// Not equal to
-    __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs)
-    {
-        return (input_itr != rhs.input_itr);
-    }
-
-    /// ostream operator
-    friend std::ostream& operator<<(std::ostream& os, const self_type& itr)
-    {
-        return os;
-    }
-};
-
-
-
-/** @} */       // end group UtilIterator
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/thread/thread_load.cuh b/pfsimulator/third_party/cub/thread/thread_load.cuh
deleted file mode 100644
index b1ca412fa..000000000
--- a/pfsimulator/third_party/cub/thread/thread_load.cuh
+++ /dev/null
@@ -1,438 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Thread utilities for reading memory using PTX cache modifiers.
- */
-
-#pragma once
-
-#include <cuda.h>
-
-#include <iterator>
-
-#include "../util_ptx.cuh"
-#include "../util_type.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \addtogroup UtilIo
- * @{
- */
-
-//-----------------------------------------------------------------------------
-// Tags and constants
-//-----------------------------------------------------------------------------
-
-/**
- * \brief Enumeration of cache modifiers for memory load operations.
- */
-enum CacheLoadModifier
-{
-    LOAD_DEFAULT,       ///< Default (no modifier)
-    LOAD_CA,            ///< Cache at all levels
-    LOAD_CG,            ///< Cache at global level
-    LOAD_CS,            ///< Cache streaming (likely to be accessed once)
-    LOAD_CV,            ///< Cache as volatile (including cached system lines)
-    LOAD_LDG,           ///< Cache as texture
-    LOAD_VOLATILE,      ///< Volatile (any memory space)
-};
-
-
-/**
- * \name Thread I/O (cache modified)
- * @{
- */
-
-/**
- * \brief Thread utility for reading memory using cub::CacheLoadModifier cache modifiers.  Can be used to load any data type.
- *
- * \par Example
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/thread/thread_load.cuh>
- *
- * // 32-bit load using cache-global modifier:
- * int *d_in;
- * int val = cub::ThreadLoad<cub::LOAD_CA>(d_in + threadIdx.x);
- *
- * // 16-bit load using default modifier
- * short *d_in;
- * short val = cub::ThreadLoad<cub::LOAD_DEFAULT>(d_in + threadIdx.x);
- *
- * // 256-bit load using cache-volatile modifier
- * double4 *d_in;
- * double4 val = cub::ThreadLoad<cub::LOAD_CV>(d_in + threadIdx.x);
- *
- * // 96-bit load using cache-streaming modifier
- * struct TestFoo { bool a; short b; };
- * TestFoo *d_struct;
- * TestFoo val = cub::ThreadLoad<cub::LOAD_CS>(d_in + threadIdx.x);
- * \endcode
- *
- * \tparam MODIFIER             <b>[inferred]</b> CacheLoadModifier enumeration
- * \tparam InputIteratorT       <b>[inferred]</b> Input iterator type \iterator
- */
-template <
-    CacheLoadModifier MODIFIER,
-    typename InputIteratorT>
-__device__ __forceinline__ typename std::iterator_traits<InputIteratorT>::value_type ThreadLoad(InputIteratorT itr);
-
-
-//@}  end member group
-
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-
-/// Helper structure for templated load iteration (inductive case)
-template <int COUNT, int MAX>
-struct IterateThreadLoad
-{
-    template <CacheLoadModifier MODIFIER, typename T>
-    static __device__ __forceinline__ void Load(T const *ptr, T *vals)
-    {
-        vals[COUNT] = ThreadLoad<MODIFIER>(ptr + COUNT);
-        IterateThreadLoad<COUNT + 1, MAX>::template Load<MODIFIER>(ptr, vals);
-    }
-
-    template <typename InputIteratorT, typename T>
-    static __device__ __forceinline__ void Dereference(InputIteratorT itr, T *vals)
-    {
-        vals[COUNT] = itr[COUNT];
-        IterateThreadLoad<COUNT + 1, MAX>::Dereference(itr, vals);
-    }
-};
-
-
-/// Helper structure for templated load iteration (termination case)
-template <int MAX>
-struct IterateThreadLoad<MAX, MAX>
-{
-    template <CacheLoadModifier MODIFIER, typename T>
-    static __device__ __forceinline__ void Load(T const * /*ptr*/, T * /*vals*/) {}
-
-    template <typename InputIteratorT, typename T>
-    static __device__ __forceinline__ void Dereference(InputIteratorT /*itr*/, T * /*vals*/) {}
-};
-
-
-/**
- * Define a uint4 (16B) ThreadLoad specialization for the given Cache load modifier
- */
-#define _CUB_LOAD_16(cub_modifier, ptx_modifier)                                             \
-    template<>                                                                              \
-    __device__ __forceinline__ uint4 ThreadLoad<cub_modifier, uint4 const *>(uint4 const *ptr)                   \
-    {                                                                                       \
-        uint4 retval;                                                                       \
-        asm volatile ("ld."#ptx_modifier".v4.u32 {%0, %1, %2, %3}, [%4];" :                 \
-            "=r"(retval.x),                                                                 \
-            "=r"(retval.y),                                                                 \
-            "=r"(retval.z),                                                                 \
-            "=r"(retval.w) :                                                                \
-            _CUB_ASM_PTR_(ptr));                                                            \
-        return retval;                                                                      \
-    }                                                                                       \
-    template<>                                                                              \
-    __device__ __forceinline__ ulonglong2 ThreadLoad<cub_modifier, ulonglong2 const *>(ulonglong2 const *ptr)    \
-    {                                                                                       \
-        ulonglong2 retval;                                                                  \
-        asm volatile ("ld."#ptx_modifier".v2.u64 {%0, %1}, [%2];" :                         \
-            "=l"(retval.x),                                                                 \
-            "=l"(retval.y) :                                                                \
-            _CUB_ASM_PTR_(ptr));                                                            \
-        return retval;                                                                      \
-    }
-
-/**
- * Define a uint2 (8B) ThreadLoad specialization for the given Cache load modifier
- */
-#define _CUB_LOAD_8(cub_modifier, ptx_modifier)                                              \
-    template<>                                                                              \
-    __device__ __forceinline__ ushort4 ThreadLoad<cub_modifier, ushort4 const *>(ushort4 const *ptr)             \
-    {                                                                                       \
-        ushort4 retval;                                                                     \
-        asm volatile ("ld."#ptx_modifier".v4.u16 {%0, %1, %2, %3}, [%4];" :                 \
-            "=h"(retval.x),                                                                 \
-            "=h"(retval.y),                                                                 \
-            "=h"(retval.z),                                                                 \
-            "=h"(retval.w) :                                                                \
-            _CUB_ASM_PTR_(ptr));                                                            \
-        return retval;                                                                      \
-    }                                                                                       \
-    template<>                                                                              \
-    __device__ __forceinline__ uint2 ThreadLoad<cub_modifier, uint2 const *>(uint2 const *ptr)                   \
-    {                                                                                       \
-        uint2 retval;                                                                       \
-        asm volatile ("ld."#ptx_modifier".v2.u32 {%0, %1}, [%2];" :                         \
-            "=r"(retval.x),                                                                 \
-            "=r"(retval.y) :                                                                \
-            _CUB_ASM_PTR_(ptr));                                                            \
-        return retval;                                                                      \
-    }                                                                                       \
-    template<>                                                                              \
-    __device__ __forceinline__ unsigned long long ThreadLoad<cub_modifier, unsigned long long const *>(unsigned long long const *ptr)    \
-    {                                                                                       \
-        unsigned long long retval;                                                          \
-        asm volatile ("ld."#ptx_modifier".u64 %0, [%1];" :                                  \
-            "=l"(retval) :                                                                  \
-            _CUB_ASM_PTR_(ptr));                                                            \
-        return retval;                                                                      \
-    }
-
-/**
- * Define a uint (4B) ThreadLoad specialization for the given Cache load modifier
- */
-#define _CUB_LOAD_4(cub_modifier, ptx_modifier)                                              \
-    template<>                                                                              \
-    __device__ __forceinline__ unsigned int ThreadLoad<cub_modifier, unsigned int const *>(unsigned int const *ptr)                      \
-    {                                                                                       \
-        unsigned int retval;                                                                \
-        asm volatile ("ld."#ptx_modifier".u32 %0, [%1];" :                                  \
-            "=r"(retval) :                                                                  \
-            _CUB_ASM_PTR_(ptr));                                                            \
-        return retval;                                                                      \
-    }
-
-
-/**
- * Define a unsigned short (2B) ThreadLoad specialization for the given Cache load modifier
- */
-#define _CUB_LOAD_2(cub_modifier, ptx_modifier)                                              \
-    template<>                                                                              \
-    __device__ __forceinline__ unsigned short ThreadLoad<cub_modifier, unsigned short const *>(unsigned short const *ptr)                \
-    {                                                                                       \
-        unsigned short retval;                                                              \
-        asm volatile ("ld."#ptx_modifier".u16 %0, [%1];" :                                  \
-            "=h"(retval) :                                                                  \
-            _CUB_ASM_PTR_(ptr));                                                            \
-        return retval;                                                                      \
-    }
-
-
-/**
- * Define an unsigned char (1B) ThreadLoad specialization for the given Cache load modifier
- */
-#define _CUB_LOAD_1(cub_modifier, ptx_modifier)                                              \
-    template<>                                                                              \
-    __device__ __forceinline__ unsigned char ThreadLoad<cub_modifier, unsigned char const *>(unsigned char const *ptr)                   \
-    {                                                                                       \
-        unsigned short retval;                                                              \
-        asm volatile (                                                                      \
-        "{"                                                                                 \
-        "   .reg .u8 datum;"                                                                \
-        "    ld."#ptx_modifier".u8 datum, [%1];"                                            \
-        "    cvt.u16.u8 %0, datum;"                                                         \
-        "}" :                                                                               \
-            "=h"(retval) :                                                                  \
-            _CUB_ASM_PTR_(ptr));                                                            \
-        return (unsigned char) retval;                                                      \
-    }
-
-
-/**
- * Define powers-of-two ThreadLoad specializations for the given Cache load modifier
- */
-#define _CUB_LOAD_ALL(cub_modifier, ptx_modifier)                                            \
-    _CUB_LOAD_16(cub_modifier, ptx_modifier)                                                 \
-    _CUB_LOAD_8(cub_modifier, ptx_modifier)                                                  \
-    _CUB_LOAD_4(cub_modifier, ptx_modifier)                                                  \
-    _CUB_LOAD_2(cub_modifier, ptx_modifier)                                                  \
-    _CUB_LOAD_1(cub_modifier, ptx_modifier)                                                  \
-
-
-/**
- * Define powers-of-two ThreadLoad specializations for the various Cache load modifiers
- */
-#if CUB_PTX_ARCH >= 200
-    _CUB_LOAD_ALL(LOAD_CA, ca)
-    _CUB_LOAD_ALL(LOAD_CG, cg)
-    _CUB_LOAD_ALL(LOAD_CS, cs)
-    _CUB_LOAD_ALL(LOAD_CV, cv)
-#else
-    _CUB_LOAD_ALL(LOAD_CA, global)
-    // Use volatile to ensure coherent reads when this PTX is JIT'd to run on newer architectures with L1
-    _CUB_LOAD_ALL(LOAD_CG, volatile.global)
-    _CUB_LOAD_ALL(LOAD_CS, global)
-    _CUB_LOAD_ALL(LOAD_CV, volatile.global)
-#endif
-
-#if CUB_PTX_ARCH >= 350
-    _CUB_LOAD_ALL(LOAD_LDG, global.nc)
-#else
-    _CUB_LOAD_ALL(LOAD_LDG, global)
-#endif
-
-
-// Macro cleanup
-#undef _CUB_LOAD_ALL
-#undef _CUB_LOAD_1
-#undef _CUB_LOAD_2
-#undef _CUB_LOAD_4
-#undef _CUB_LOAD_8
-#undef _CUB_LOAD_16
-
-
-
-/**
- * ThreadLoad definition for LOAD_DEFAULT modifier on iterator types
- */
-template <typename InputIteratorT>
-__device__ __forceinline__ typename std::iterator_traits<InputIteratorT>::value_type ThreadLoad(
-    InputIteratorT          itr,
-    Int2Type<LOAD_DEFAULT>  /*modifier*/,
-    Int2Type<false>         /*is_pointer*/)
-{
-    return *itr;
-}
-
-
-/**
- * ThreadLoad definition for LOAD_DEFAULT modifier on pointer types
- */
-template <typename T>
-__device__ __forceinline__ T ThreadLoad(
-    T                       *ptr,
-    Int2Type<LOAD_DEFAULT>  /*modifier*/,
-    Int2Type<true>          /*is_pointer*/)
-{
-    return *ptr;
-}
-
-
-/**
- * ThreadLoad definition for LOAD_VOLATILE modifier on primitive pointer types
- */
-template <typename T>
-__device__ __forceinline__ T ThreadLoadVolatilePointer(
-    T                       *ptr,
-    Int2Type<true>          /*is_primitive*/)
-{
-    T retval = *reinterpret_cast<volatile T*>(ptr);
-    return retval;
-}
-
-
-/**
- * ThreadLoad definition for LOAD_VOLATILE modifier on non-primitive pointer types
- */
-template <typename T>
-__device__ __forceinline__ T ThreadLoadVolatilePointer(
-    T                       *ptr,
-    Int2Type<false>         /*is_primitive*/)
-{
-    typedef typename UnitWord<T>::VolatileWord VolatileWord;   // Word type for memcopying
-
-    const int VOLATILE_MULTIPLE = sizeof(T) / sizeof(VolatileWord);
-/*
-    VolatileWord words[VOLATILE_MULTIPLE];
-
-    IterateThreadLoad<0, VOLATILE_MULTIPLE>::Dereference(
-        reinterpret_cast<volatile VolatileWord*>(ptr),
-        words);
-
-    return *reinterpret_cast<T*>(words);
-*/
-
-    T retval;
-    VolatileWord *words = reinterpret_cast<VolatileWord*>(&retval);
-    IterateThreadLoad<0, VOLATILE_MULTIPLE>::Dereference(
-        reinterpret_cast<volatile VolatileWord*>(ptr),
-        words);
-    return retval;
-}
-
-
-/**
- * ThreadLoad definition for LOAD_VOLATILE modifier on pointer types
- */
-template <typename T>
-__device__ __forceinline__ T ThreadLoad(
-    T                       *ptr,
-    Int2Type<LOAD_VOLATILE> /*modifier*/,
-    Int2Type<true>          /*is_pointer*/)
-{
-    // Apply tags for partial-specialization
-    return ThreadLoadVolatilePointer(ptr, Int2Type<Traits<T>::PRIMITIVE>());
-}
-
-
-/**
- * ThreadLoad definition for generic modifiers on pointer types
- */
-template <typename T, int MODIFIER>
-__device__ __forceinline__ T ThreadLoad(
-    T const                 *ptr,
-    Int2Type<MODIFIER>      /*modifier*/,
-    Int2Type<true>          /*is_pointer*/)
-{
-    typedef typename UnitWord<T>::DeviceWord DeviceWord;
-
-    const int DEVICE_MULTIPLE = sizeof(T) / sizeof(DeviceWord);
-
-    DeviceWord words[DEVICE_MULTIPLE];
-
-    IterateThreadLoad<0, DEVICE_MULTIPLE>::template Load<CacheLoadModifier(MODIFIER)>(
-        reinterpret_cast<DeviceWord*>(const_cast<T*>(ptr)),
-        words);
-
-    return *reinterpret_cast<T*>(words);
-}
-
-
-/**
- * ThreadLoad definition for generic modifiers
- */
-template <
-    CacheLoadModifier MODIFIER,
-    typename InputIteratorT>
-__device__ __forceinline__ typename std::iterator_traits<InputIteratorT>::value_type ThreadLoad(InputIteratorT itr)
-{
-    // Apply tags for partial-specialization
-    return ThreadLoad(
-        itr,
-        Int2Type<MODIFIER>(),
-        Int2Type<IsPointer<InputIteratorT>::VALUE>());
-}
-
-
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-/** @} */       // end group UtilIo
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/thread/thread_operators.cuh b/pfsimulator/third_party/cub/thread/thread_operators.cuh
deleted file mode 100644
index 76cd800f5..000000000
--- a/pfsimulator/third_party/cub/thread/thread_operators.cuh
+++ /dev/null
@@ -1,317 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Simple binary operator functor types
- */
-
-/******************************************************************************
- * Simple functor operators
- ******************************************************************************/
-
-#pragma once
-
-#include "../util_macro.cuh"
-#include "../util_type.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \addtogroup UtilModule
- * @{
- */
-
-/**
- * \brief Default equality functor
- */
-struct Equality
-{
-    /// Boolean equality operator, returns <tt>(a == b)</tt>
-    template <typename T>
-    __host__ __device__ __forceinline__ bool operator()(const T &a, const T &b) const
-    {
-        return a == b;
-    }
-};
-
-
-/**
- * \brief Default inequality functor
- */
-struct Inequality
-{
-    /// Boolean inequality operator, returns <tt>(a != b)</tt>
-    template <typename T>
-    __host__ __device__ __forceinline__ bool operator()(const T &a, const T &b) const
-    {
-        return a != b;
-    }
-};
-
-
-/**
- * \brief Inequality functor (wraps equality functor)
- */
-template <typename EqualityOp>
-struct InequalityWrapper
-{
-    /// Wrapped equality operator
-    EqualityOp op;
-
-    /// Constructor
-    __host__ __device__ __forceinline__
-    InequalityWrapper(EqualityOp op) : op(op) {}
-
-    /// Boolean inequality operator, returns <tt>(a != b)</tt>
-    template <typename T>
-    __host__ __device__ __forceinline__ bool operator()(const T &a, const T &b)
-    {
-        return !op(a, b);
-    }
-};
-
-
-/**
- * \brief Default sum functor
- */
-struct Sum
-{
-    /// Boolean sum operator, returns <tt>a + b</tt>
-    template <typename T>
-    __host__ __device__ __forceinline__ T operator()(const T &a, const T &b) const
-    {
-        return a + b;
-    }
-};
-
-
-/**
- * \brief Default max functor
- */
-struct Max
-{
-    /// Boolean max operator, returns <tt>(a > b) ? a : b</tt>
-    template <typename T>
-    __host__ __device__ __forceinline__ T operator()(const T &a, const T &b) const
-    {
-        return CUB_MAX(a, b);
-    }
-};
-
-
-/**
- * \brief Arg max functor (keeps the value and offset of the first occurrence of the larger item)
- */
-struct ArgMax
-{
-    /// Boolean max operator, preferring the item having the smaller offset in case of ties
-    template <typename T, typename OffsetT>
-    __host__ __device__ __forceinline__ KeyValuePair<OffsetT, T> operator()(
-        const KeyValuePair<OffsetT, T> &a,
-        const KeyValuePair<OffsetT, T> &b) const
-    {
-// Mooch BUG (device reduce argmax gk110 3.2 million random fp32)
-//        return ((b.value > a.value) || ((a.value == b.value) && (b.key < a.key))) ? b : a;
-
-        if ((b.value > a.value) || ((a.value == b.value) && (b.key < a.key)))
-            return b;
-        return a;
-    }
-};
-
-
-/**
- * \brief Default min functor
- */
-struct Min
-{
-    /// Boolean min operator, returns <tt>(a < b) ? a : b</tt>
-    template <typename T>
-    __host__ __device__ __forceinline__ T operator()(const T &a, const T &b) const
-    {
-        return CUB_MIN(a, b);
-    }
-};
-
-
-/**
- * \brief Arg min functor (keeps the value and offset of the first occurrence of the smallest item)
- */
-struct ArgMin
-{
-    /// Boolean min operator, preferring the item having the smaller offset in case of ties
-    template <typename T, typename OffsetT>
-    __host__ __device__ __forceinline__ KeyValuePair<OffsetT, T> operator()(
-        const KeyValuePair<OffsetT, T> &a,
-        const KeyValuePair<OffsetT, T> &b) const
-    {
-// Mooch BUG (device reduce argmax gk110 3.2 million random fp32)
-//        return ((b.value < a.value) || ((a.value == b.value) && (b.key < a.key))) ? b : a;
-
-        if ((b.value < a.value) || ((a.value == b.value) && (b.key < a.key)))
-            return b;
-        return a;
-    }
-};
-
-
-/**
- * \brief Default cast functor
- */
-template <typename B>
-struct CastOp
-{
-    /// Cast operator, returns <tt>(B) a</tt>
-    template <typename A>
-    __host__ __device__ __forceinline__ B operator()(const A &a) const
-    {
-        return (B) a;
-    }
-};
-
-
-/**
- * \brief Binary operator wrapper for switching non-commutative scan arguments
- */
-template <typename ScanOp>
-class SwizzleScanOp
-{
-private:
-
-    /// Wrapped scan operator
-    ScanOp scan_op;
-
-public:
-
-    /// Constructor
-    __host__ __device__ __forceinline__
-    SwizzleScanOp(ScanOp scan_op) : scan_op(scan_op) {}
-
-    /// Switch the scan arguments
-    template <typename T>
-    __host__ __device__ __forceinline__
-    T operator()(const T &a, const T &b)
-    {
-      T _a(a);
-      T _b(b);
-
-      return scan_op(_b, _a);
-    }
-};
-
-
-/**
- * \brief Reduce-by-segment functor.
- *
- * Given two cub::KeyValuePair inputs \p a and \p b and a
- * binary associative combining operator \p <tt>f(const T &x, const T &y)</tt>,
- * an instance of this functor returns a cub::KeyValuePair whose \p key
- * field is <tt>a.key</tt> + <tt>b.key</tt>, and whose \p value field
- * is either b.value if b.key is non-zero, or f(a.value, b.value) otherwise.
- *
- * ReduceBySegmentOp is an associative, non-commutative binary combining operator
- * for input sequences of cub::KeyValuePair pairings.  Such
- * sequences are typically used to represent a segmented set of values to be reduced
- * and a corresponding set of {0,1}-valued integer "head flags" demarcating the
- * first value of each segment.
- *
- */
-template <typename ReductionOpT>    ///< Binary reduction operator to apply to values
-struct ReduceBySegmentOp
-{
-    /// Wrapped reduction operator
-    ReductionOpT op;
-
-    /// Constructor
-    __host__ __device__ __forceinline__ ReduceBySegmentOp() {}
-
-    /// Constructor
-    __host__ __device__ __forceinline__ ReduceBySegmentOp(ReductionOpT op) : op(op) {}
-
-    /// Scan operator
-    template <typename KeyValuePairT>       ///< KeyValuePair pairing of T (value) and OffsetT (head flag)
-    __host__ __device__ __forceinline__ KeyValuePairT operator()(
-        const KeyValuePairT &first,         ///< First partial reduction
-        const KeyValuePairT &second)        ///< Second partial reduction
-    {
-        KeyValuePairT retval;
-        retval.key = first.key + second.key;
-        retval.value = (second.key) ?
-                second.value :                          // The second partial reduction spans a segment reset, so it's value aggregate becomes the running aggregate
-                op(first.value, second.value);          // The second partial reduction does not span a reset, so accumulate both into the running aggregate
-        return retval;
-    }
-};
-
-
-
-template <typename ReductionOpT>    ///< Binary reduction operator to apply to values
-struct ReduceByKeyOp
-{
-    /// Wrapped reduction operator
-    ReductionOpT op;
-
-    /// Constructor
-    __host__ __device__ __forceinline__ ReduceByKeyOp() {}
-
-    /// Constructor
-    __host__ __device__ __forceinline__ ReduceByKeyOp(ReductionOpT op) : op(op) {}
-
-    /// Scan operator
-    template <typename KeyValuePairT>
-    __host__ __device__ __forceinline__ KeyValuePairT operator()(
-        const KeyValuePairT &first,       ///< First partial reduction
-        const KeyValuePairT &second)      ///< Second partial reduction
-    {
-        KeyValuePairT retval = second;
-
-        if (first.key == second.key)
-            retval.value = op(first.value, retval.value);
-
-        return retval;
-    }
-};
-
-
-
-
-
-
-
-/** @} */       // end group UtilModule
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/thread/thread_reduce.cuh b/pfsimulator/third_party/cub/thread/thread_reduce.cuh
deleted file mode 100644
index 4c13688f3..000000000
--- a/pfsimulator/third_party/cub/thread/thread_reduce.cuh
+++ /dev/null
@@ -1,152 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Thread utilities for sequential reduction over statically-sized array types
- */
-
-#pragma once
-
-#include "../thread/thread_operators.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/// Internal namespace (to prevent ADL mishaps between static functions when mixing different CUB installations)
-namespace internal {
-
-/**
- * Sequential reduction over statically-sized array types
- */
-template <
-    int         LENGTH,
-    typename    T,
-    typename    ReductionOp>
-__device__ __forceinline__ T ThreadReduce(
-    T*                  input,                  ///< [in] Input array
-    ReductionOp         reduction_op,           ///< [in] Binary reduction operator
-    T                   prefix,                 ///< [in] Prefix to seed reduction with
-    Int2Type<LENGTH>    /*length*/)
-{
-    T retval = prefix;
-
-    #pragma unroll
-    for (int i = 0; i < LENGTH; ++i)
-        retval = reduction_op(retval, input[i]);
-
-    return retval;
-}
-
-
-/**
- * \brief Perform a sequential reduction over \p LENGTH elements of the \p input array, seeded with the specified \p prefix.  The aggregate is returned.
- *
- * \tparam LENGTH     LengthT of input array
- * \tparam T          <b>[inferred]</b> The data type to be reduced.
- * \tparam ScanOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
- */
-template <
-    int         LENGTH,
-    typename    T,
-    typename    ReductionOp>
-__device__ __forceinline__ T ThreadReduce(
-    T*          input,                  ///< [in] Input array
-    ReductionOp reduction_op,           ///< [in] Binary reduction operator
-    T           prefix)                 ///< [in] Prefix to seed reduction with
-{
-    return ThreadReduce(input, reduction_op, prefix, Int2Type<LENGTH>());
-}
-
-
-/**
- * \brief Perform a sequential reduction over \p LENGTH elements of the \p input array.  The aggregate is returned.
- *
- * \tparam LENGTH     LengthT of input array
- * \tparam T          <b>[inferred]</b> The data type to be reduced.
- * \tparam ScanOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
- */
-template <
-    int         LENGTH,
-    typename    T,
-    typename    ReductionOp>
-__device__ __forceinline__ T ThreadReduce(
-    T*          input,                  ///< [in] Input array
-    ReductionOp reduction_op)           ///< [in] Binary reduction operator
-{
-    T prefix = input[0];
-    return ThreadReduce<LENGTH - 1>(input + 1, reduction_op, prefix);
-}
-
-
-/**
- * \brief Perform a sequential reduction over the statically-sized \p input array, seeded with the specified \p prefix.  The aggregate is returned.
- *
- * \tparam LENGTH     <b>[inferred]</b> LengthT of \p input array
- * \tparam T          <b>[inferred]</b> The data type to be reduced.
- * \tparam ScanOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
- */
-template <
-    int         LENGTH,
-    typename    T,
-    typename    ReductionOp>
-__device__ __forceinline__ T ThreadReduce(
-    T           (&input)[LENGTH],       ///< [in] Input array
-    ReductionOp reduction_op,           ///< [in] Binary reduction operator
-    T           prefix)                 ///< [in] Prefix to seed reduction with
-{
-    return ThreadReduce(input, reduction_op, prefix, Int2Type<LENGTH>());
-}
-
-
-/**
- * \brief Serial reduction with the specified operator
- *
- * \tparam LENGTH     <b>[inferred]</b> LengthT of \p input array
- * \tparam T          <b>[inferred]</b> The data type to be reduced.
- * \tparam ScanOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
- */
-template <
-    int         LENGTH,
-    typename    T,
-    typename    ReductionOp>
-__device__ __forceinline__ T ThreadReduce(
-    T           (&input)[LENGTH],       ///< [in] Input array
-    ReductionOp reduction_op)           ///< [in] Binary reduction operator
-{
-    return ThreadReduce<LENGTH>((T*) input, reduction_op);
-}
-
-
-}               // internal namespace
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/thread/thread_scan.cuh b/pfsimulator/third_party/cub/thread/thread_scan.cuh
deleted file mode 100644
index 8d67549ae..000000000
--- a/pfsimulator/third_party/cub/thread/thread_scan.cuh
+++ /dev/null
@@ -1,268 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Thread utilities for sequential prefix scan over statically-sized array types
- */
-
-#pragma once
-
-#include "../thread/thread_operators.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/// Internal namespace (to prevent ADL mishaps between static functions when mixing different CUB installations)
-namespace internal {
-
-
-/**
- * \addtogroup UtilModule
- * @{
- */
-
-/**
- * \name Sequential prefix scan over statically-sized array types
- * @{
- */
-
-template <
-    int         LENGTH,
-    typename    T,
-    typename    ScanOp>
-__device__ __forceinline__ T ThreadScanExclusive(
-    T                   inclusive,
-    T                   exclusive,
-    T                   *input,                 ///< [in] Input array
-    T                   *output,                ///< [out] Output array (may be aliased to \p input)
-    ScanOp              scan_op,                ///< [in] Binary scan operator
-    Int2Type<LENGTH>    /*length*/)
-{
-    #pragma unroll
-    for (int i = 0; i < LENGTH; ++i)
-    {
-        inclusive = scan_op(exclusive, input[i]);
-        output[i] = exclusive;
-        exclusive = inclusive;
-    }
-
-    return inclusive;
-}
-
-
-
-/**
- * \brief Perform a sequential exclusive prefix scan over \p LENGTH elements of the \p input array, seeded with the specified \p prefix.  The aggregate is returned.
- *
- * \tparam LENGTH     LengthT of \p input and \p output arrays
- * \tparam T          <b>[inferred]</b> The data type to be scanned.
- * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
- */
-template <
-    int         LENGTH,
-    typename    T,
-    typename    ScanOp>
-__device__ __forceinline__ T ThreadScanExclusive(
-    T           *input,                 ///< [in] Input array
-    T           *output,                ///< [out] Output array (may be aliased to \p input)
-    ScanOp      scan_op,                ///< [in] Binary scan operator
-    T           prefix,                 ///< [in] Prefix to seed scan with
-    bool        apply_prefix = true)    ///< [in] Whether or not the calling thread should apply its prefix.  If not, the first output element is undefined.  (Handy for preventing thread-0 from applying a prefix.)
-{
-    T inclusive = input[0];
-    if (apply_prefix)
-    {
-        inclusive = scan_op(prefix, inclusive);
-    }
-    output[0] = prefix;
-    T exclusive = inclusive;
-
-    return ThreadScanExclusive(inclusive, exclusive, input + 1, output + 1, scan_op, Int2Type<LENGTH - 1>());
-}
-
-
-/**
- * \brief Perform a sequential exclusive prefix scan over the statically-sized \p input array, seeded with the specified \p prefix.  The aggregate is returned.
- *
- * \tparam LENGTH     <b>[inferred]</b> LengthT of \p input and \p output arrays
- * \tparam T          <b>[inferred]</b> The data type to be scanned.
- * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
- */
-template <
-    int         LENGTH,
-    typename    T,
-    typename    ScanOp>
-__device__ __forceinline__ T ThreadScanExclusive(
-    T           (&input)[LENGTH],       ///< [in] Input array
-    T           (&output)[LENGTH],      ///< [out] Output array (may be aliased to \p input)
-    ScanOp      scan_op,                ///< [in] Binary scan operator
-    T           prefix,                 ///< [in] Prefix to seed scan with
-    bool        apply_prefix = true)    ///< [in] Whether or not the calling thread should apply its prefix.  (Handy for preventing thread-0 from applying a prefix.)
-{
-    return ThreadScanExclusive<LENGTH>((T*) input, (T*) output, scan_op, prefix, apply_prefix);
-}
-
-
-
-
-
-
-
-
-
-template <
-    int         LENGTH,
-    typename    T,
-    typename    ScanOp>
-__device__ __forceinline__ T ThreadScanInclusive(
-    T                   inclusive,
-    T                   *input,                 ///< [in] Input array
-    T                   *output,                ///< [out] Output array (may be aliased to \p input)
-    ScanOp              scan_op,                ///< [in] Binary scan operator
-    Int2Type<LENGTH>    /*length*/)
-{
-    #pragma unroll
-    for (int i = 0; i < LENGTH; ++i)
-    {
-        inclusive = scan_op(inclusive, input[i]);
-        output[i] = inclusive;
-    }
-
-    return inclusive;
-}
-
-
-/**
- * \brief Perform a sequential inclusive prefix scan over \p LENGTH elements of the \p input array.  The aggregate is returned.
- *
- * \tparam LENGTH     LengthT of \p input and \p output arrays
- * \tparam T          <b>[inferred]</b> The data type to be scanned.
- * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
- */
-template <
-    int         LENGTH,
-    typename    T,
-    typename    ScanOp>
-__device__ __forceinline__ T ThreadScanInclusive(
-    T           *input,                 ///< [in] Input array
-    T           *output,                ///< [out] Output array (may be aliased to \p input)
-    ScanOp      scan_op)                ///< [in] Binary scan operator
-{
-    T inclusive = input[0];
-    output[0] = inclusive;
-
-    // Continue scan
-    return ThreadScanInclusive(inclusive, input + 1, output + 1, scan_op, Int2Type<LENGTH - 1>());
-}
-
-
-/**
- * \brief Perform a sequential inclusive prefix scan over the statically-sized \p input array.  The aggregate is returned.
- *
- * \tparam LENGTH     <b>[inferred]</b> LengthT of \p input and \p output arrays
- * \tparam T          <b>[inferred]</b> The data type to be scanned.
- * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
- */
-template <
-    int         LENGTH,
-    typename    T,
-    typename    ScanOp>
-__device__ __forceinline__ T ThreadScanInclusive(
-    T           (&input)[LENGTH],       ///< [in] Input array
-    T           (&output)[LENGTH],      ///< [out] Output array (may be aliased to \p input)
-    ScanOp      scan_op)                ///< [in] Binary scan operator
-{
-    return ThreadScanInclusive<LENGTH>((T*) input, (T*) output, scan_op);
-}
-
-
-/**
- * \brief Perform a sequential inclusive prefix scan over \p LENGTH elements of the \p input array, seeded with the specified \p prefix.  The aggregate is returned.
- *
- * \tparam LENGTH     LengthT of \p input and \p output arrays
- * \tparam T          <b>[inferred]</b> The data type to be scanned.
- * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
- */
-template <
-    int         LENGTH,
-    typename    T,
-    typename    ScanOp>
-__device__ __forceinline__ T ThreadScanInclusive(
-    T           *input,                 ///< [in] Input array
-    T           *output,                ///< [out] Output array (may be aliased to \p input)
-    ScanOp      scan_op,                ///< [in] Binary scan operator
-    T           prefix,                 ///< [in] Prefix to seed scan with
-    bool        apply_prefix = true)    ///< [in] Whether or not the calling thread should apply its prefix.  (Handy for preventing thread-0 from applying a prefix.)
-{
-    T inclusive = input[0];
-    if (apply_prefix)
-    {
-        inclusive = scan_op(prefix, inclusive);
-    }
-    output[0] = inclusive;
-
-    // Continue scan
-    return ThreadScanInclusive(inclusive, input + 1, output + 1, scan_op, Int2Type<LENGTH - 1>());
-}
-
-
-/**
- * \brief Perform a sequential inclusive prefix scan over the statically-sized \p input array, seeded with the specified \p prefix.  The aggregate is returned.
- *
- * \tparam LENGTH     <b>[inferred]</b> LengthT of \p input and \p output arrays
- * \tparam T          <b>[inferred]</b> The data type to be scanned.
- * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
- */
-template <
-    int         LENGTH,
-    typename    T,
-    typename    ScanOp>
-__device__ __forceinline__ T ThreadScanInclusive(
-    T           (&input)[LENGTH],       ///< [in] Input array
-    T           (&output)[LENGTH],      ///< [out] Output array (may be aliased to \p input)
-    ScanOp      scan_op,                ///< [in] Binary scan operator
-    T           prefix,                 ///< [in] Prefix to seed scan with
-    bool        apply_prefix = true)    ///< [in] Whether or not the calling thread should apply its prefix.  (Handy for preventing thread-0 from applying a prefix.)
-{
-    return ThreadScanInclusive<LENGTH>((T*) input, (T*) output, scan_op, prefix, apply_prefix);
-}
-
-
-//@}  end member group
-
-/** @} */       // end group UtilModule
-
-
-}               // internal namespace
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/thread/thread_search.cuh b/pfsimulator/third_party/cub/thread/thread_search.cuh
deleted file mode 100644
index 3099080a3..000000000
--- a/pfsimulator/third_party/cub/thread/thread_search.cuh
+++ /dev/null
@@ -1,154 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Thread utilities for sequential search
- */
-
-#pragma once
-
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * Computes the begin offsets into A and B for the specific diagonal
- */
-template <
-    typename AIteratorT,
-    typename BIteratorT,
-    typename OffsetT,
-    typename CoordinateT>
-__host__ __device__ __forceinline__ void MergePathSearch(
-    OffsetT         diagonal,
-    AIteratorT      a,
-    BIteratorT      b,
-    OffsetT         a_len,
-    OffsetT         b_len,
-    CoordinateT&    path_coordinate)
-{
-    /// The value type of the input iterator
-    typedef typename std::iterator_traits<AIteratorT>::value_type T;
-
-    OffsetT split_min = CUB_MAX(diagonal - b_len, 0);
-    OffsetT split_max = CUB_MIN(diagonal, a_len);
-
-    while (split_min < split_max)
-    {
-        OffsetT split_pivot = (split_min + split_max) >> 1;
-        if (a[split_pivot] <= b[diagonal - split_pivot - 1])
-        {
-            // Move candidate split range up A, down B
-            split_min = split_pivot + 1;
-        }
-        else
-        {
-            // Move candidate split range up B, down A
-            split_max = split_pivot;
-        }
-    }
-
-    path_coordinate.x = CUB_MIN(split_min, a_len);
-    path_coordinate.y = diagonal - split_min;
-}
-
-
-
-/**
- * \brief Returns the offset of the first value within \p input which does not compare less than \p val
- */
-template <
-    typename InputIteratorT,
-    typename OffsetT,
-    typename T>
-__device__ __forceinline__ OffsetT LowerBound(
-    InputIteratorT      input,              ///< [in] Input sequence
-    OffsetT             num_items,          ///< [in] Input sequence length
-    T                   val)                ///< [in] Search key
-{
-    OffsetT retval = 0;
-    while (num_items > 0)
-    {
-        OffsetT half = num_items >> 1;
-        if (input[retval + half] < val)
-        {
-            retval = retval + (half + 1);
-            num_items = num_items - (half + 1);
-        }
-        else
-        {
-            num_items = half;
-        }
-    }
-
-    return retval;
-}
-
-
-/**
- * \brief Returns the offset of the first value within \p input which compares greater than \p val
- */
-template <
-    typename InputIteratorT,
-    typename OffsetT,
-    typename T>
-__device__ __forceinline__ OffsetT UpperBound(
-    InputIteratorT      input,              ///< [in] Input sequence
-    OffsetT             num_items,          ///< [in] Input sequence length
-    T                   val)                ///< [in] Search key
-{
-    OffsetT retval = 0;
-    while (num_items > 0)
-    {
-        OffsetT half = num_items >> 1;
-        if (val < input[retval + half])
-        {
-            num_items = half;
-        }
-        else
-        {
-            retval = retval + (half + 1);
-            num_items = num_items - (half + 1);
-        }
-    }
-
-    return retval;
-}
-
-
-
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/thread/thread_store.cuh b/pfsimulator/third_party/cub/thread/thread_store.cuh
deleted file mode 100644
index ec20b36f4..000000000
--- a/pfsimulator/third_party/cub/thread/thread_store.cuh
+++ /dev/null
@@ -1,422 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Thread utilities for writing memory using PTX cache modifiers.
- */
-
-#pragma once
-
-#include <cuda.h>
-
-#include "../util_ptx.cuh"
-#include "../util_type.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \addtogroup UtilIo
- * @{
- */
-
-
-//-----------------------------------------------------------------------------
-// Tags and constants
-//-----------------------------------------------------------------------------
-
-/**
- * \brief Enumeration of cache modifiers for memory store operations.
- */
-enum CacheStoreModifier
-{
-    STORE_DEFAULT,              ///< Default (no modifier)
-    STORE_WB,                   ///< Cache write-back all coherent levels
-    STORE_CG,                   ///< Cache at global level
-    STORE_CS,                   ///< Cache streaming (likely to be accessed once)
-    STORE_WT,                   ///< Cache write-through (to system memory)
-    STORE_VOLATILE,             ///< Volatile shared (any memory space)
-};
-
-
-/**
- * \name Thread I/O (cache modified)
- * @{
- */
-
-/**
- * \brief Thread utility for writing memory using cub::CacheStoreModifier cache modifiers.  Can be used to store any data type.
- *
- * \par Example
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/thread/thread_store.cuh>
- *
- * // 32-bit store using cache-global modifier:
- * int *d_out;
- * int val;
- * cub::ThreadStore<cub::STORE_CG>(d_out + threadIdx.x, val);
- *
- * // 16-bit store using default modifier
- * short *d_out;
- * short val;
- * cub::ThreadStore<cub::STORE_DEFAULT>(d_out + threadIdx.x, val);
- *
- * // 256-bit store using write-through modifier
- * double4 *d_out;
- * double4 val;
- * cub::ThreadStore<cub::STORE_WT>(d_out + threadIdx.x, val);
- *
- * // 96-bit store using cache-streaming cache modifier
- * struct TestFoo { bool a; short b; };
- * TestFoo *d_struct;
- * TestFoo val;
- * cub::ThreadStore<cub::STORE_CS>(d_out + threadIdx.x, val);
- * \endcode
- *
- * \tparam MODIFIER             <b>[inferred]</b> CacheStoreModifier enumeration
- * \tparam InputIteratorT       <b>[inferred]</b> Output iterator type \iterator
- * \tparam T                    <b>[inferred]</b> Data type of output value
- */
-template <
-    CacheStoreModifier  MODIFIER,
-    typename            OutputIteratorT,
-    typename            T>
-__device__ __forceinline__ void ThreadStore(OutputIteratorT itr, T val);
-
-
-//@}  end member group
-
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-
-/// Helper structure for templated store iteration (inductive case)
-template <int COUNT, int MAX>
-struct IterateThreadStore
-{
-    template <CacheStoreModifier MODIFIER, typename T>
-    static __device__ __forceinline__ void Store(T *ptr, T *vals)
-    {
-        ThreadStore<MODIFIER>(ptr + COUNT, vals[COUNT]);
-        IterateThreadStore<COUNT + 1, MAX>::template Store<MODIFIER>(ptr, vals);
-    }
-
-    template <typename OutputIteratorT, typename T>
-    static __device__ __forceinline__ void Dereference(OutputIteratorT ptr, T *vals)
-    {
-        ptr[COUNT] = vals[COUNT];
-        IterateThreadStore<COUNT + 1, MAX>::Dereference(ptr, vals);
-    }
-
-};
-
-/// Helper structure for templated store iteration (termination case)
-template <int MAX>
-struct IterateThreadStore<MAX, MAX>
-{
-    template <CacheStoreModifier MODIFIER, typename T>
-    static __device__ __forceinline__ void Store(T * /*ptr*/, T * /*vals*/) {}
-
-    template <typename OutputIteratorT, typename T>
-    static __device__ __forceinline__ void Dereference(OutputIteratorT /*ptr*/, T * /*vals*/) {}
-};
-
-
-/**
- * Define a uint4 (16B) ThreadStore specialization for the given Cache load modifier
- */
-#define _CUB_STORE_16(cub_modifier, ptx_modifier)                                            \
-    template<>                                                                              \
-    __device__ __forceinline__ void ThreadStore<cub_modifier, uint4*, uint4>(uint4* ptr, uint4 val)                         \
-    {                                                                                       \
-        asm volatile ("st."#ptx_modifier".v4.u32 [%0], {%1, %2, %3, %4};" : :               \
-            _CUB_ASM_PTR_(ptr),                                                             \
-            "r"(val.x),                                                                     \
-            "r"(val.y),                                                                     \
-            "r"(val.z),                                                                     \
-            "r"(val.w));                                                                    \
-    }                                                                                       \
-    template<>                                                                              \
-    __device__ __forceinline__ void ThreadStore<cub_modifier, ulonglong2*, ulonglong2>(ulonglong2* ptr, ulonglong2 val)     \
-    {                                                                                       \
-        asm volatile ("st."#ptx_modifier".v2.u64 [%0], {%1, %2};" : :                       \
-            _CUB_ASM_PTR_(ptr),                                                             \
-            "l"(val.x),                                                                     \
-            "l"(val.y));                                                                    \
-    }
-
-
-/**
- * Define a uint2 (8B) ThreadStore specialization for the given Cache load modifier
- */
-#define _CUB_STORE_8(cub_modifier, ptx_modifier)                                             \
-    template<>                                                                              \
-    __device__ __forceinline__ void ThreadStore<cub_modifier, ushort4*, ushort4>(ushort4* ptr, ushort4 val)                 \
-    {                                                                                       \
-        asm volatile ("st."#ptx_modifier".v4.u16 [%0], {%1, %2, %3, %4};" : :               \
-            _CUB_ASM_PTR_(ptr),                                                             \
-            "h"(val.x),                                                                     \
-            "h"(val.y),                                                                     \
-            "h"(val.z),                                                                     \
-            "h"(val.w));                                                                    \
-    }                                                                                       \
-    template<>                                                                              \
-    __device__ __forceinline__ void ThreadStore<cub_modifier, uint2*, uint2>(uint2* ptr, uint2 val)                         \
-    {                                                                                       \
-        asm volatile ("st."#ptx_modifier".v2.u32 [%0], {%1, %2};" : :                       \
-            _CUB_ASM_PTR_(ptr),                                                             \
-            "r"(val.x),                                                                     \
-            "r"(val.y));                                                                    \
-    }                                                                                       \
-    template<>                                                                              \
-    __device__ __forceinline__ void ThreadStore<cub_modifier, unsigned long long*, unsigned long long>(unsigned long long* ptr, unsigned long long val)     \
-    {                                                                                       \
-        asm volatile ("st."#ptx_modifier".u64 [%0], %1;" : :                                \
-            _CUB_ASM_PTR_(ptr),                                                             \
-            "l"(val));                                                                      \
-    }
-
-/**
- * Define a unsigned int (4B) ThreadStore specialization for the given Cache load modifier
- */
-#define _CUB_STORE_4(cub_modifier, ptx_modifier)                                             \
-    template<>                                                                              \
-    __device__ __forceinline__ void ThreadStore<cub_modifier, unsigned int*, unsigned int>(unsigned int* ptr, unsigned int val)                             \
-    {                                                                                       \
-        asm volatile ("st."#ptx_modifier".u32 [%0], %1;" : :                                \
-            _CUB_ASM_PTR_(ptr),                                                             \
-            "r"(val));                                                                      \
-    }
-
-
-/**
- * Define a unsigned short (2B) ThreadStore specialization for the given Cache load modifier
- */
-#define _CUB_STORE_2(cub_modifier, ptx_modifier)                                             \
-    template<>                                                                              \
-    __device__ __forceinline__ void ThreadStore<cub_modifier, unsigned short*, unsigned short>(unsigned short* ptr, unsigned short val)                     \
-    {                                                                                       \
-        asm volatile ("st."#ptx_modifier".u16 [%0], %1;" : :                                \
-            _CUB_ASM_PTR_(ptr),                                                             \
-            "h"(val));                                                                      \
-    }
-
-
-/**
- * Define a unsigned char (1B) ThreadStore specialization for the given Cache load modifier
- */
-#define _CUB_STORE_1(cub_modifier, ptx_modifier)                                             \
-    template<>                                                                              \
-    __device__ __forceinline__ void ThreadStore<cub_modifier, unsigned char*, unsigned char>(unsigned char* ptr, unsigned char val)                         \
-    {                                                                                       \
-        asm volatile (                                                                      \
-        "{"                                                                                 \
-        "   .reg .u8 datum;"                                                                \
-        "   cvt.u8.u16 datum, %1;"                                                          \
-        "   st."#ptx_modifier".u8 [%0], datum;"                                             \
-        "}" : :                                                                             \
-            _CUB_ASM_PTR_(ptr),                                                             \
-            "h"((unsigned short) val));                                                               \
-    }
-
-/**
- * Define powers-of-two ThreadStore specializations for the given Cache load modifier
- */
-#define _CUB_STORE_ALL(cub_modifier, ptx_modifier)                                           \
-    _CUB_STORE_16(cub_modifier, ptx_modifier)                                                \
-    _CUB_STORE_8(cub_modifier, ptx_modifier)                                                 \
-    _CUB_STORE_4(cub_modifier, ptx_modifier)                                                 \
-    _CUB_STORE_2(cub_modifier, ptx_modifier)                                                 \
-    _CUB_STORE_1(cub_modifier, ptx_modifier)                                                 \
-
-
-/**
- * Define ThreadStore specializations for the various Cache load modifiers
- */
-#if CUB_PTX_ARCH >= 200
-    _CUB_STORE_ALL(STORE_WB, wb)
-    _CUB_STORE_ALL(STORE_CG, cg)
-    _CUB_STORE_ALL(STORE_CS, cs)
-    _CUB_STORE_ALL(STORE_WT, wt)
-#else
-    _CUB_STORE_ALL(STORE_WB, global)
-    _CUB_STORE_ALL(STORE_CG, global)
-    _CUB_STORE_ALL(STORE_CS, global)
-    _CUB_STORE_ALL(STORE_WT, volatile.global)
-#endif
-
-
-// Macro cleanup
-#undef _CUB_STORE_ALL
-#undef _CUB_STORE_1
-#undef _CUB_STORE_2
-#undef _CUB_STORE_4
-#undef _CUB_STORE_8
-#undef _CUB_STORE_16
-
-
-/**
- * ThreadStore definition for STORE_DEFAULT modifier on iterator types
- */
-template <typename OutputIteratorT, typename T>
-__device__ __forceinline__ void ThreadStore(
-    OutputIteratorT             itr,
-    T                           val,
-    Int2Type<STORE_DEFAULT>     /*modifier*/,
-    Int2Type<false>             /*is_pointer*/)
-{
-    *itr = val;
-}
-
-
-/**
- * ThreadStore definition for STORE_DEFAULT modifier on pointer types
- */
-template <typename T>
-__device__ __forceinline__ void ThreadStore(
-    T                           *ptr,
-    T                           val,
-    Int2Type<STORE_DEFAULT>     /*modifier*/,
-    Int2Type<true>              /*is_pointer*/)
-{
-    *ptr = val;
-}
-
-
-/**
- * ThreadStore definition for STORE_VOLATILE modifier on primitive pointer types
- */
-template <typename T>
-__device__ __forceinline__ void ThreadStoreVolatilePtr(
-    T                           *ptr,
-    T                           val,
-    Int2Type<true>              /*is_primitive*/)
-{
-    *reinterpret_cast<volatile T*>(ptr) = val;
-}
-
-
-/**
- * ThreadStore definition for STORE_VOLATILE modifier on non-primitive pointer types
- */
-template <typename T>
-__device__ __forceinline__ void ThreadStoreVolatilePtr(
-    T                           *ptr,
-    T                           val,
-    Int2Type<false>             /*is_primitive*/)
-{
-    // Create a temporary using shuffle-words, then store using volatile-words
-    typedef typename UnitWord<T>::VolatileWord  VolatileWord;  
-    typedef typename UnitWord<T>::ShuffleWord   ShuffleWord;
-
-    const int VOLATILE_MULTIPLE = sizeof(T) / sizeof(VolatileWord);
-    const int SHUFFLE_MULTIPLE  = sizeof(T) / sizeof(ShuffleWord);
-    
-    VolatileWord words[VOLATILE_MULTIPLE];
-
-    #pragma unroll
-    for (int i = 0; i < SHUFFLE_MULTIPLE; ++i)
-        reinterpret_cast<ShuffleWord*>(words)[i] = reinterpret_cast<ShuffleWord*>(&val)[i];
-
-    IterateThreadStore<0, VOLATILE_MULTIPLE>::template Dereference(
-        reinterpret_cast<volatile VolatileWord*>(ptr),
-        words);
-}
-
-
-/**
- * ThreadStore definition for STORE_VOLATILE modifier on pointer types
- */
-template <typename T>
-__device__ __forceinline__ void ThreadStore(
-    T                           *ptr,
-    T                           val,
-    Int2Type<STORE_VOLATILE>    /*modifier*/,
-    Int2Type<true>              /*is_pointer*/)
-{
-    ThreadStoreVolatilePtr(ptr, val, Int2Type<Traits<T>::PRIMITIVE>());
-}
-
-
-/**
- * ThreadStore definition for generic modifiers on pointer types
- */
-template <typename T, int MODIFIER>
-__device__ __forceinline__ void ThreadStore(
-    T                           *ptr,
-    T                           val,
-    Int2Type<MODIFIER>          /*modifier*/,
-    Int2Type<true>              /*is_pointer*/)
-{
-    // Create a temporary using shuffle-words, then store using device-words
-    typedef typename UnitWord<T>::DeviceWord    DeviceWord;  
-    typedef typename UnitWord<T>::ShuffleWord   ShuffleWord;
-
-    const int DEVICE_MULTIPLE   = sizeof(T) / sizeof(DeviceWord);
-    const int SHUFFLE_MULTIPLE  = sizeof(T) / sizeof(ShuffleWord);
-    
-    DeviceWord words[DEVICE_MULTIPLE];
-
-    #pragma unroll
-    for (int i = 0; i < SHUFFLE_MULTIPLE; ++i)
-        reinterpret_cast<ShuffleWord*>(words)[i] = reinterpret_cast<ShuffleWord*>(&val)[i];
-
-    IterateThreadStore<0, DEVICE_MULTIPLE>::template Store<CacheStoreModifier(MODIFIER)>(
-        reinterpret_cast<DeviceWord*>(ptr),
-        words);
-}
-
-
-/**
- * ThreadStore definition for generic modifiers
- */
-template <CacheStoreModifier MODIFIER, typename OutputIteratorT, typename T>
-__device__ __forceinline__ void ThreadStore(OutputIteratorT itr, T val)
-{
-    ThreadStore(
-        itr,
-        val,
-        Int2Type<MODIFIER>(),
-        Int2Type<IsPointer<OutputIteratorT>::VALUE>());
-}
-
-
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-/** @} */       // end group UtilIo
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/util_allocator.cuh b/pfsimulator/third_party/cub/util_allocator.cuh
deleted file mode 100644
index 0e6dd0486..000000000
--- a/pfsimulator/third_party/cub/util_allocator.cuh
+++ /dev/null
@@ -1,708 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/******************************************************************************
- * Simple caching allocator for device memory allocations. The allocator is
- * thread-safe and capable of managing device allocations on multiple devices.
- ******************************************************************************/
-
-#pragma once
-
-#include "util_namespace.cuh"
-#include "util_debug.cuh"
-
-#include <set>
-#include <map>
-
-#include "host/mutex.cuh"
-#include <math.h>
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \addtogroup UtilMgmt
- * @{
- */
-
-
-/******************************************************************************
- * CachingDeviceAllocator (host use)
- ******************************************************************************/
-
-/**
- * \brief A simple caching allocator for device memory allocations.
- *
- * \par Overview
- * The allocator is thread-safe and stream-safe and is capable of managing cached
- * device allocations on multiple devices.  It behaves as follows:
- *
- * \par
- * - Allocations from the allocator are associated with an \p active_stream.  Once freed,
- *   the allocation becomes available immediately for reuse within the \p active_stream
- *   with which it was associated with during allocation, and it becomes available for
- *   reuse within other streams when all prior work submitted to \p active_stream has completed.
- * - Allocations are categorized and cached by bin size.  A new allocation request of
- *   a given size will only consider cached allocations within the corresponding bin.
- * - Bin limits progress geometrically in accordance with the growth factor
- *   \p bin_growth provided during construction.  Unused device allocations within
- *   a larger bin cache are not reused for allocation requests that categorize to
- *   smaller bin sizes.
- * - Allocation requests below (\p bin_growth ^ \p min_bin) are rounded up to
- *   (\p bin_growth ^ \p min_bin).
- * - Allocations above (\p bin_growth ^ \p max_bin) are not rounded up to the nearest
- *   bin and are simply freed when they are deallocated instead of being returned
- *   to a bin-cache.
- * - %If the total storage of cached allocations on a given device will exceed
- *   \p max_cached_bytes, allocations for that device are simply freed when they are
- *   deallocated instead of being returned to their bin-cache.
- *
- * \par
- * For example, the default-constructed CachingDeviceAllocator is configured with:
- * - \p bin_growth          = 8
- * - \p min_bin             = 3
- * - \p max_bin             = 7
- * - \p max_cached_bytes    = 6MB - 1B
- *
- * \par
- * which delineates five bin-sizes: 512B, 4KB, 32KB, 256KB, and 2MB
- * and sets a maximum of 6,291,455 cached bytes per device
- *
- */
-struct CachingDeviceAllocator
-{
-
-    //---------------------------------------------------------------------
-    // Constants
-    //---------------------------------------------------------------------
-
-    /// Out-of-bounds bin
-    static const unsigned int INVALID_BIN = (unsigned int) -1;
-
-    /// Invalid size
-    static const size_t INVALID_SIZE = (size_t) -1;
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-    /// Invalid device ordinal
-    static const int INVALID_DEVICE_ORDINAL = -1;
-
-    //---------------------------------------------------------------------
-    // Type definitions and helper types
-    //---------------------------------------------------------------------
-
-    /**
-     * Descriptor for device memory allocations
-     */
-    struct BlockDescriptor
-    {
-        void*           d_ptr;              // Device pointer
-        size_t          bytes;              // Size of allocation in bytes
-        unsigned int    bin;                // Bin enumeration
-        int             device;             // device ordinal
-        cudaStream_t    associated_stream;  // Associated associated_stream
-        cudaEvent_t     ready_event;        // Signal when associated stream has run to the point at which this block was freed
-
-        // Constructor (suitable for searching maps for a specific block, given its pointer and device)
-        BlockDescriptor(void *d_ptr, int device) :
-            d_ptr(d_ptr),
-            bytes(0),
-            bin(INVALID_BIN),
-            device(device),
-            associated_stream(0),
-            ready_event(0)
-        {}
-
-        // Constructor (suitable for searching maps for a range of suitable blocks, given a device)
-        BlockDescriptor(int device) :
-            d_ptr(NULL),
-            bytes(0),
-            bin(INVALID_BIN),
-            device(device),
-            associated_stream(0),
-            ready_event(0)
-        {}
-
-        // Comparison functor for comparing device pointers
-        static bool PtrCompare(const BlockDescriptor &a, const BlockDescriptor &b)
-        {
-            if (a.device == b.device)
-                return (a.d_ptr < b.d_ptr);
-            else
-                return (a.device < b.device);
-        }
-
-        // Comparison functor for comparing allocation sizes
-        static bool SizeCompare(const BlockDescriptor &a, const BlockDescriptor &b)
-        {
-            if (a.device == b.device)
-                return (a.bytes < b.bytes);
-            else
-                return (a.device < b.device);
-        }
-    };
-
-    /// BlockDescriptor comparator function interface
-    typedef bool (*Compare)(const BlockDescriptor &, const BlockDescriptor &);
-
-    class TotalBytes {
-    public:
-        size_t free;
-        size_t live;
-        TotalBytes() { free = live = 0; }
-    };
-
-    /// Set type for cached blocks (ordered by size)
-    typedef std::multiset<BlockDescriptor, Compare> CachedBlocks;
-
-    /// Set type for live blocks (ordered by ptr)
-    typedef std::multiset<BlockDescriptor, Compare> BusyBlocks;
-
-    /// Map type of device ordinals to the number of cached bytes cached by each device
-    typedef std::map<int, TotalBytes> GpuCachedBytes;
-
-
-    //---------------------------------------------------------------------
-    // Utility functions
-    //---------------------------------------------------------------------
-
-    /**
-     * Integer pow function for unsigned base and exponent
-     */
-    static unsigned int IntPow(
-        unsigned int base,
-        unsigned int exp)
-    {
-        unsigned int retval = 1;
-        while (exp > 0)
-        {
-            if (exp & 1) {
-                retval = retval * base;        // multiply the result by the current base
-            }
-            base = base * base;                // square the base
-            exp = exp >> 1;                    // divide the exponent in half
-        }
-        return retval;
-    }
-
-
-    /**
-     * Round up to the nearest power-of
-     */
-    void NearestPowerOf(
-        unsigned int    &power,
-        size_t          &rounded_bytes,
-        unsigned int    base,
-        size_t          value)
-    {
-        power = 0;
-        rounded_bytes = 1;
-
-        if (value * base < value)
-        {
-            // Overflow
-            power = sizeof(size_t) * 8;
-            rounded_bytes = size_t(0) - 1;
-            return;
-        }
-
-        while (rounded_bytes < value)
-        {
-            rounded_bytes *= base;
-            power++;
-        }
-    }
-
-
-    //---------------------------------------------------------------------
-    // Fields
-    //---------------------------------------------------------------------
-
-    cub::Mutex      mutex;              /// Mutex for thread-safety
-
-    unsigned int    bin_growth;         /// Geometric growth factor for bin-sizes
-    unsigned int    min_bin;            /// Minimum bin enumeration
-    unsigned int    max_bin;            /// Maximum bin enumeration
-
-    size_t          min_bin_bytes;      /// Minimum bin size
-    size_t          max_bin_bytes;      /// Maximum bin size
-    size_t          max_cached_bytes;   /// Maximum aggregate cached bytes per device
-
-    const bool      skip_cleanup;       /// Whether or not to skip a call to FreeAllCached() when destructor is called.  (The CUDA runtime may have already shut down for statically declared allocators)
-    bool            debug;              /// Whether or not to print (de)allocation events to stdout
-
-    GpuCachedBytes  cached_bytes;       /// Map of device ordinal to aggregate cached bytes on that device
-    CachedBlocks    cached_blocks;      /// Set of cached device allocations available for reuse
-    BusyBlocks      live_blocks;        /// Set of live device allocations currently in use
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-    //---------------------------------------------------------------------
-    // Methods
-    //---------------------------------------------------------------------
-
-    /**
-     * \brief Constructor.
-     */
-    CachingDeviceAllocator(
-        unsigned int    bin_growth,                             ///< Geometric growth factor for bin-sizes
-        unsigned int    min_bin             = 1,                ///< Minimum bin (default is bin_growth ^ 1)
-        unsigned int    max_bin             = INVALID_BIN,      ///< Maximum bin (default is no max bin)
-        size_t          max_cached_bytes    = INVALID_SIZE,     ///< Maximum aggregate cached bytes per device (default is no limit)
-        bool            skip_cleanup        = false,            ///< Whether or not to skip a call to \p FreeAllCached() when the destructor is called (default is to deallocate)
-        bool            debug               = false)            ///< Whether or not to print (de)allocation events to stdout (default is no stderr output)
-    :
-        bin_growth(bin_growth),
-        min_bin(min_bin),
-        max_bin(max_bin),
-        min_bin_bytes(IntPow(bin_growth, min_bin)),
-        max_bin_bytes(IntPow(bin_growth, max_bin)),
-        max_cached_bytes(max_cached_bytes),
-        skip_cleanup(skip_cleanup),
-        debug(debug),
-        cached_blocks(BlockDescriptor::SizeCompare),
-        live_blocks(BlockDescriptor::PtrCompare)
-    {}
-
-
-    /**
-     * \brief Default constructor.
-     *
-     * Configured with:
-     * \par
-     * - \p bin_growth          = 8
-     * - \p min_bin             = 3
-     * - \p max_bin             = 7
-     * - \p max_cached_bytes    = (\p bin_growth ^ \p max_bin) * 3) - 1 = 6,291,455 bytes
-     *
-     * which delineates five bin-sizes: 512B, 4KB, 32KB, 256KB, and 2MB and
-     * sets a maximum of 6,291,455 cached bytes per device
-     */
-    CachingDeviceAllocator(
-        bool skip_cleanup = false,
-        bool debug = false)
-    :
-        bin_growth(8),
-        min_bin(3),
-        max_bin(7),
-        min_bin_bytes(IntPow(bin_growth, min_bin)),
-        max_bin_bytes(IntPow(bin_growth, max_bin)),
-        max_cached_bytes((max_bin_bytes * 3) - 1),
-        skip_cleanup(skip_cleanup),
-        debug(debug),
-        cached_blocks(BlockDescriptor::SizeCompare),
-        live_blocks(BlockDescriptor::PtrCompare)
-    {}
-
-
-    /**
-     * \brief Sets the limit on the number bytes this allocator is allowed to cache per device.
-     *
-     * Changing the ceiling of cached bytes does not cause any allocations (in-use or
-     * cached-in-reserve) to be freed.  See \p FreeAllCached().
-     */
-    cudaError_t SetMaxCachedBytes(
-        size_t max_cached_bytes)
-    {
-        // Lock
-        mutex.Lock();
-
-        if (debug) _CubLog("Changing max_cached_bytes (%lld -> %lld)\n", (long long) this->max_cached_bytes, (long long) max_cached_bytes);
-
-        this->max_cached_bytes = max_cached_bytes;
-
-        // Unlock
-        mutex.Unlock();
-
-        return cudaSuccess;
-    }
-
-
-    /**
-     * \brief Provides a suitable allocation of device memory for the given size on the specified device.
-     *
-     * Once freed, the allocation becomes available immediately for reuse within the \p active_stream
-     * with which it was associated with during allocation, and it becomes available for reuse within other
-     * streams when all prior work submitted to \p active_stream has completed.
-     */
-    cudaError_t DeviceAllocate(
-        int             device,             ///< [in] Device on which to place the allocation
-        void            **d_ptr,            ///< [out] Reference to pointer to the allocation
-        size_t          bytes,              ///< [in] Minimum number of bytes for the allocation
-        cudaStream_t    active_stream = 0)  ///< [in] The stream to be associated with this allocation
-    {
-        *d_ptr                          = NULL;
-        int entrypoint_device           = INVALID_DEVICE_ORDINAL;
-        cudaError_t error               = cudaSuccess;
-
-        if (device == INVALID_DEVICE_ORDINAL)
-        {
-            if (CubDebug(error = cudaGetDevice(&entrypoint_device))) return error;
-            device = entrypoint_device;
-        }
-
-        // Create a block descriptor for the requested allocation
-        bool found = false;
-        BlockDescriptor search_key(device);
-        search_key.associated_stream = active_stream;
-        NearestPowerOf(search_key.bin, search_key.bytes, bin_growth, bytes);
-
-        if (search_key.bin > max_bin)
-        {
-            // Bin is greater than our maximum bin: allocate the request
-            // exactly and give out-of-bounds bin.  It will not be cached
-            // for reuse when returned.
-            search_key.bin      = INVALID_BIN;
-            search_key.bytes    = bytes;
-        }
-        else
-        {
-            // Search for a suitable cached allocation: lock
-            mutex.Lock();
-
-            if (search_key.bin < min_bin)
-            {
-                // Bin is less than minimum bin: round up
-                search_key.bin      = min_bin;
-                search_key.bytes    = min_bin_bytes;
-            }
-
-            // Iterate through the range of cached blocks on the same device in the same bin
-            CachedBlocks::iterator block_itr = cached_blocks.lower_bound(search_key);
-            while ((block_itr != cached_blocks.end())
-                    && (block_itr->device == device)
-                    && (block_itr->bin == search_key.bin))
-            {
-                // To prevent races with reusing blocks returned by the host but still
-                // in use by the device, only consider cached blocks that are
-                // either (from the active stream) or (from an idle stream)
-                if ((active_stream == block_itr->associated_stream) ||
-                    (cudaEventQuery(block_itr->ready_event) != cudaErrorNotReady))
-                {
-                    // Reuse existing cache block.  Insert into live blocks.
-                    found = true;
-                    search_key = *block_itr;
-                    search_key.associated_stream = active_stream;
-                    live_blocks.insert(search_key);
-
-                    // Remove from free blocks
-                    cached_bytes[device].free -= search_key.bytes;
-                    cached_bytes[device].live += search_key.bytes;
-
-                    if (debug) _CubLog("\tDevice %d reused cached block at %p (%lld bytes) for stream %lld (previously associated with stream %lld).\n",
-                        device, search_key.d_ptr, (long long) search_key.bytes, (long long) search_key.associated_stream, (long long)  block_itr->associated_stream);
-
-                    cached_blocks.erase(block_itr);
-
-                    break;
-                }
-                block_itr++;
-            }
-
-            // Done searching: unlock
-            mutex.Unlock();
-        }
-
-        // Allocate the block if necessary
-        if (!found)
-        {
-            // Set runtime's current device to specified device (entrypoint may not be set)
-            if (device != entrypoint_device)
-            {
-                if (CubDebug(error = cudaGetDevice(&entrypoint_device))) return error;
-                if (CubDebug(error = cudaSetDevice(device))) return error;
-            }
-
-            // Attempt to allocate
-            if (CubDebug(error = cudaMalloc(&search_key.d_ptr, search_key.bytes)) == cudaErrorMemoryAllocation)
-            {
-                // The allocation attempt failed: free all cached blocks on device and retry
-                if (debug) _CubLog("\tDevice %d failed to allocate %lld bytes for stream %lld, retrying after freeing cached allocations",
-                      device, (long long) search_key.bytes, (long long) search_key.associated_stream);
-
-                error = cudaSuccess;    // Reset the error we will return
-                cudaGetLastError();     // Reset CUDART's error
-
-                // Lock
-                mutex.Lock();
-
-                // Iterate the range of free blocks on the same device
-                BlockDescriptor free_key(device);
-                CachedBlocks::iterator block_itr = cached_blocks.lower_bound(free_key);
-
-                while ((block_itr != cached_blocks.end()) && (block_itr->device == device))
-                {
-                    // No need to worry about synchronization with the device: cudaFree is
-                    // blocking and will synchronize across all kernels executing
-                    // on the current device
-
-                    // Free device memory and destroy stream event.
-                    if (CubDebug(error = cudaFree(block_itr->d_ptr))) break;
-                    if (CubDebug(error = cudaEventDestroy(block_itr->ready_event))) break;
-
-                    // Reduce balance and erase entry
-                    cached_bytes[device].free -= block_itr->bytes;
-
-                    if (debug) _CubLog("\tDevice %d freed %lld bytes.\n\t\t  %lld available blocks cached (%lld bytes), %lld live blocks (%lld bytes) outstanding.\n",
-                        device, (long long) block_itr->bytes, (long long) cached_blocks.size(), (long long) cached_bytes[device].free, (long long) live_blocks.size(), (long long) cached_bytes[device].live);
-
-                    cached_blocks.erase(block_itr);
-
-                    block_itr++;
-                }
-
-                // Unlock
-                mutex.Unlock();
-
-                // Return under error
-                if (error) return error;
-
-                // Try to allocate again
-                if (CubDebug(error = cudaMalloc(&search_key.d_ptr, search_key.bytes))) return error;
-            }
-
-            // Create ready event
-            if (CubDebug(error = cudaEventCreateWithFlags(&search_key.ready_event, cudaEventDisableTiming)))
-                return error;
-
-            // Insert into live blocks
-            mutex.Lock();
-            live_blocks.insert(search_key);
-            cached_bytes[device].live += search_key.bytes;
-            mutex.Unlock();
-
-            if (debug) _CubLog("\tDevice %d allocated new device block at %p (%lld bytes associated with stream %lld).\n",
-                      device, search_key.d_ptr, (long long) search_key.bytes, (long long) search_key.associated_stream);
-
-            // Attempt to revert back to previous device if necessary
-            if ((entrypoint_device != INVALID_DEVICE_ORDINAL) && (entrypoint_device != device))
-            {
-                if (CubDebug(error = cudaSetDevice(entrypoint_device))) return error;
-            }
-        }
-
-        // Copy device pointer to output parameter
-        *d_ptr = search_key.d_ptr;
-
-        if (debug) _CubLog("\t\t%lld available blocks cached (%lld bytes), %lld live blocks outstanding(%lld bytes).\n",
-            (long long) cached_blocks.size(), (long long) cached_bytes[device].free, (long long) live_blocks.size(), (long long) cached_bytes[device].live);
-
-        return error;
-    }
-
-
-    /**
-     * \brief Provides a suitable allocation of device memory for the given size on the current device.
-     *
-     * Once freed, the allocation becomes available immediately for reuse within the \p active_stream
-     * with which it was associated with during allocation, and it becomes available for reuse within other
-     * streams when all prior work submitted to \p active_stream has completed.
-     */
-    cudaError_t DeviceAllocate(
-        void            **d_ptr,            ///< [out] Reference to pointer to the allocation
-        size_t          bytes,              ///< [in] Minimum number of bytes for the allocation
-        cudaStream_t    active_stream = 0)  ///< [in] The stream to be associated with this allocation
-    {
-        return DeviceAllocate(INVALID_DEVICE_ORDINAL, d_ptr, bytes, active_stream);
-    }
-
-
-    /**
-     * \brief Frees a live allocation of device memory on the specified device, returning it to the allocator.
-     *
-     * Once freed, the allocation becomes available immediately for reuse within the \p active_stream
-     * with which it was associated with during allocation, and it becomes available for reuse within other
-     * streams when all prior work submitted to \p active_stream has completed.
-     */
-    cudaError_t DeviceFree(
-        int             device,
-        void*           d_ptr)
-    {
-        int entrypoint_device           = INVALID_DEVICE_ORDINAL;
-        cudaError_t error               = cudaSuccess;
-
-        if (device == INVALID_DEVICE_ORDINAL)
-        {
-            if (CubDebug(error = cudaGetDevice(&entrypoint_device)))
-                return error;
-            device = entrypoint_device;
-        }
-
-        // Lock
-        mutex.Lock();
-
-        // Find corresponding block descriptor
-        bool recached = false;
-        BlockDescriptor search_key(d_ptr, device);
-        BusyBlocks::iterator block_itr = live_blocks.find(search_key);
-        if (block_itr != live_blocks.end())
-        {
-            // Remove from live blocks
-            search_key = *block_itr;
-            live_blocks.erase(block_itr);
-            cached_bytes[device].live -= search_key.bytes;
-
-            // Keep the returned allocation if bin is valid and we won't exceed the max cached threshold
-            if ((search_key.bin != INVALID_BIN) && (cached_bytes[device].free + search_key.bytes <= max_cached_bytes))
-            {
-                // Insert returned allocation into free blocks
-                recached = true;
-                cached_blocks.insert(search_key);
-                cached_bytes[device].free += search_key.bytes;
-
-                if (debug) _CubLog("\tDevice %d returned %lld bytes from associated stream %lld.\n\t\t %lld available blocks cached (%lld bytes), %lld live blocks outstanding. (%lld bytes)\n",
-                    device, (long long) search_key.bytes, (long long) search_key.associated_stream, (long long) cached_blocks.size(),
-                    (long long) cached_bytes[device].free, (long long) live_blocks.size(), (long long) cached_bytes[device].live);
-            }
-        }
-
-        // Unlock
-        mutex.Unlock();
-
-        // First set to specified device (entrypoint may not be set)
-        if (device != entrypoint_device)
-        {
-            if (CubDebug(error = cudaGetDevice(&entrypoint_device))) return error;
-            if (CubDebug(error = cudaSetDevice(device))) return error;
-        }
-
-        if (recached)
-        {
-            // Insert the ready event in the associated stream (must have current device set properly)
-            if (CubDebug(error = cudaEventRecord(search_key.ready_event, search_key.associated_stream))) return error;
-        }
-        else
-        {
-            // Free the allocation from the runtime and cleanup the event.
-            if (CubDebug(error = cudaFree(d_ptr))) return error;
-            if (CubDebug(error = cudaEventDestroy(search_key.ready_event))) return error;
-
-            if (debug) _CubLog("\tDevice %d freed %lld bytes from associated stream %lld.\n\t\t  %lld available blocks cached (%lld bytes), %lld live blocks (%lld bytes) outstanding.\n",
-                device, (long long) search_key.bytes, (long long) search_key.associated_stream, (long long) cached_blocks.size(), (long long) cached_bytes[device].free, (long long) live_blocks.size(), (long long) cached_bytes[device].live);
-        }
-
-        // Reset device
-        if ((entrypoint_device != INVALID_DEVICE_ORDINAL) && (entrypoint_device != device))
-        {
-            if (CubDebug(error = cudaSetDevice(entrypoint_device))) return error;
-        }
-
-        return error;
-    }
-
-
-    /**
-     * \brief Frees a live allocation of device memory on the current device, returning it to the allocator.
-     *
-     * Once freed, the allocation becomes available immediately for reuse within the \p active_stream
-     * with which it was associated with during allocation, and it becomes available for reuse within other
-     * streams when all prior work submitted to \p active_stream has completed.
-     */
-    cudaError_t DeviceFree(
-        void*           d_ptr)
-    {
-        return DeviceFree(INVALID_DEVICE_ORDINAL, d_ptr);
-    }
-
-
-    /**
-     * \brief Frees all cached device allocations on all devices
-     */
-    cudaError_t FreeAllCached()
-    {
-        cudaError_t error         = cudaSuccess;
-        int entrypoint_device     = INVALID_DEVICE_ORDINAL;
-        int current_device        = INVALID_DEVICE_ORDINAL;
-
-        mutex.Lock();
-
-        while (!cached_blocks.empty())
-        {
-            // Get first block
-            CachedBlocks::iterator begin = cached_blocks.begin();
-
-            // Get entry-point device ordinal if necessary
-            if (entrypoint_device == INVALID_DEVICE_ORDINAL)
-            {
-                if (CubDebug(error = cudaGetDevice(&entrypoint_device))) break;
-            }
-
-            // Set current device ordinal if necessary
-            if (begin->device != current_device)
-            {
-                if (CubDebug(error = cudaSetDevice(begin->device))) break;
-                current_device = begin->device;
-            }
-
-            // Free device memory
-            if (CubDebug(error = cudaFree(begin->d_ptr))) break;
-            if (CubDebug(error = cudaEventDestroy(begin->ready_event))) break;
-
-            // Reduce balance and erase entry
-            cached_bytes[current_device].free -= begin->bytes;
-
-            if (debug) _CubLog("\tDevice %d freed %lld bytes.\n\t\t  %lld available blocks cached (%lld bytes), %lld live blocks (%lld bytes) outstanding.\n",
-                current_device, (long long) begin->bytes, (long long) cached_blocks.size(), (long long) cached_bytes[current_device].free, (long long) live_blocks.size(), (long long) cached_bytes[current_device].live);
-
-            cached_blocks.erase(begin);
-        }
-
-        mutex.Unlock();
-
-        // Attempt to revert back to entry-point device if necessary
-        if (entrypoint_device != INVALID_DEVICE_ORDINAL)
-        {
-            if (CubDebug(error = cudaSetDevice(entrypoint_device))) return error;
-        }
-
-        return error;
-    }
-
-
-    /**
-     * \brief Destructor
-     */
-    virtual ~CachingDeviceAllocator()
-    {
-        if (!skip_cleanup)
-            FreeAllCached();
-    }
-
-};
-
-
-
-
-/** @} */       // end group UtilMgmt
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/util_arch.cuh b/pfsimulator/third_party/cub/util_arch.cuh
deleted file mode 100644
index 28d81e7cd..000000000
--- a/pfsimulator/third_party/cub/util_arch.cuh
+++ /dev/null
@@ -1,151 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Static architectural properties by SM version.
- */
-
-#pragma once
-
-#include "util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-#if (__CUDACC_VER_MAJOR__ >= 9) && !defined(CUB_USE_COOPERATIVE_GROUPS)
-    #define CUB_USE_COOPERATIVE_GROUPS
-#endif
-
-/// CUB_PTX_ARCH reflects the PTX version targeted by the active compiler pass (or zero during the host pass).
-#ifndef CUB_PTX_ARCH
-    #ifndef __CUDA_ARCH__
-        #define CUB_PTX_ARCH 0
-    #else
-        #define CUB_PTX_ARCH __CUDA_ARCH__
-    #endif
-#endif
-
-
-/// Whether or not the source targeted by the active compiler pass is allowed to  invoke device kernels or methods from the CUDA runtime API.
-#ifndef CUB_RUNTIME_FUNCTION
-    #if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__>= 350 && defined(__CUDACC_RDC__))
-        #define CUB_RUNTIME_ENABLED
-        #define CUB_RUNTIME_FUNCTION __host__ __device__
-    #else
-        #define CUB_RUNTIME_FUNCTION __host__
-    #endif
-#endif
-
-
-/// Number of threads per warp
-#ifndef CUB_LOG_WARP_THREADS
-    #define CUB_LOG_WARP_THREADS(arch)                      \
-        (5)
-    #define CUB_WARP_THREADS(arch)                          \
-        (1 << CUB_LOG_WARP_THREADS(arch))
-
-    #define CUB_PTX_WARP_THREADS        CUB_WARP_THREADS(CUB_PTX_ARCH)
-    #define CUB_PTX_LOG_WARP_THREADS    CUB_LOG_WARP_THREADS(CUB_PTX_ARCH)
-#endif
-
-
-/// Number of smem banks
-#ifndef CUB_LOG_SMEM_BANKS
-    #define CUB_LOG_SMEM_BANKS(arch)                        \
-        ((arch >= 200) ?                                    \
-            (5) :                                           \
-            (4))
-    #define CUB_SMEM_BANKS(arch)                            \
-        (1 << CUB_LOG_SMEM_BANKS(arch))
-
-    #define CUB_PTX_LOG_SMEM_BANKS      CUB_LOG_SMEM_BANKS(CUB_PTX_ARCH)
-    #define CUB_PTX_SMEM_BANKS          CUB_SMEM_BANKS(CUB_PTX_ARCH)
-#endif
-
-
-/// Oversubscription factor
-#ifndef CUB_SUBSCRIPTION_FACTOR
-    #define CUB_SUBSCRIPTION_FACTOR(arch)                   \
-        ((arch >= 300) ?                                    \
-            (5) :                                           \
-            ((arch >= 200) ?                                \
-                (3) :                                       \
-                (10)))
-    #define CUB_PTX_SUBSCRIPTION_FACTOR             CUB_SUBSCRIPTION_FACTOR(CUB_PTX_ARCH)
-#endif
-
-
-/// Prefer padding overhead vs X-way conflicts greater than this threshold
-#ifndef CUB_PREFER_CONFLICT_OVER_PADDING
-    #define CUB_PREFER_CONFLICT_OVER_PADDING(arch)          \
-        ((arch >= 300) ?                                    \
-            (1) :                                           \
-            (4))
-    #define CUB_PTX_PREFER_CONFLICT_OVER_PADDING    CUB_PREFER_CONFLICT_OVER_PADDING(CUB_PTX_ARCH)
-#endif
-
-
-/// Scale down the number of threads to keep same amount of scratch storage as the nominal configuration for 4B data.  Minimum of two warps.
-#ifndef CUB_SCALED_BLOCK_THREADS
-    #define CUB_SCALED_BLOCK_THREADS(NOMINAL_4B_BLOCK_THREADS, T, PTX_ARCH)                   \
-        (CUB_MIN(                                                                           \
-            NOMINAL_4B_BLOCK_THREADS,                                                       \
-            CUB_WARP_THREADS(PTX_ARCH) * CUB_MAX(                                           \
-                2,                                                                          \
-                (NOMINAL_4B_BLOCK_THREADS / CUB_WARP_THREADS(PTX_ARCH)) * 4 / sizeof(T))))
-#endif
-
-/// Scale down number of items per thread to keep the same amount of register storage as the nominal configuration for 4B data.  Minimum 1 item per thread
-#ifndef CUB_SCALED_ITEMS_PER_THREAD
-    #define CUB_SCALED_ITEMS_PER_THREAD(NOMINAL_4B_ITEMS_PER_THREAD, NOMINAL_4B_BLOCK_THREADS, T, PTX_ARCH)     \
-        CUB_MAX(                                                                                                \
-            1,                                                                                                  \
-            (sizeof(T) < 4) ?                                                                                   \
-                ((NOMINAL_4B_ITEMS_PER_THREAD * NOMINAL_4B_BLOCK_THREADS * 4) / CUB_MAX(4, sizeof(T))) / CUB_SCALED_BLOCK_THREADS(NOMINAL_4B_BLOCK_THREADS, T, PTX_ARCH) / 2 :  \
-                ((NOMINAL_4B_ITEMS_PER_THREAD * NOMINAL_4B_BLOCK_THREADS * 4) / CUB_MAX(4, sizeof(T))) / CUB_SCALED_BLOCK_THREADS(NOMINAL_4B_BLOCK_THREADS, T, PTX_ARCH))
-#endif
-
-/// Define both nominal threads-per-block and items-per-thread
-#ifndef CUB_SCALED_GRANULARITIES
-    #define CUB_SCALED_GRANULARITIES(NOMINAL_4B_BLOCK_THREADS, NOMINAL_4B_ITEMS_PER_THREAD, T)      \
-        CUB_SCALED_BLOCK_THREADS(NOMINAL_4B_BLOCK_THREADS, T, 200),                                   \
-        CUB_SCALED_ITEMS_PER_THREAD(NOMINAL_4B_ITEMS_PER_THREAD, NOMINAL_4B_BLOCK_THREADS, T, 200)
-#endif
-
-
-
-#endif  // Do not document
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/util_debug.cuh b/pfsimulator/third_party/cub/util_debug.cuh
deleted file mode 100644
index 3ad832e73..000000000
--- a/pfsimulator/third_party/cub/util_debug.cuh
+++ /dev/null
@@ -1,145 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Error and event logging routines.
- *
- * The following macros definitions are supported:
- * - \p CUB_LOG.  Simple event messages are printed to \p stdout.
- */
-
-#pragma once
-
-#include <stdio.h>
-#include "util_namespace.cuh"
-#include "util_arch.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \addtogroup UtilMgmt
- * @{
- */
-
-
-/// CUB error reporting macro (prints error messages to stderr)
-#if (defined(DEBUG) || defined(_DEBUG)) && !defined(CUB_STDERR)
-    #define CUB_STDERR
-#endif
-
-
-
-/**
- * \brief %If \p CUB_STDERR is defined and \p error is not \p cudaSuccess, the corresponding error message is printed to \p stderr (or \p stdout in device code) along with the supplied source context.
- *
- * \return The CUDA error.
- */
-__host__ __device__ __forceinline__ cudaError_t Debug(
-    cudaError_t     error,
-    const char*     filename,
-    int             line)
-{
-    (void)filename;
-    (void)line;
-#ifdef CUB_STDERR
-    if (error)
-    {
-    #if (CUB_PTX_ARCH == 0)
-        fprintf(stderr, "CUDA error %d [%s, %d]: %s\n", error, filename, line, cudaGetErrorString(error));
-        fflush(stderr);
-    #elif (CUB_PTX_ARCH >= 200)
-        printf("CUDA error %d [block (%d,%d,%d) thread (%d,%d,%d), %s, %d]\n", error, blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, filename, line);
-    #endif
-    }
-#endif
-    return error;
-}
-
-
-/**
- * \brief Debug macro
- */
-#ifndef CubDebug
-    #define CubDebug(e) cub::Debug((cudaError_t) (e), __FILE__, __LINE__)
-#endif
-
-
-/**
- * \brief Debug macro with exit
- */
-#ifndef CubDebugExit
-    #define CubDebugExit(e) if (cub::Debug((cudaError_t) (e), __FILE__, __LINE__)) { exit(1); }
-#endif
-
-
-/**
- * \brief Log macro for printf statements.
- */
-#if !defined(_CubLog)
-    #if !(defined(__clang__) && defined(__CUDA__))
-        #if (CUB_PTX_ARCH == 0)
-            #define _CubLog(format, ...) printf(format,__VA_ARGS__);
-        #elif (CUB_PTX_ARCH >= 200)
-            #define _CubLog(format, ...) printf("[block (%d,%d,%d), thread (%d,%d,%d)]: " format, blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, __VA_ARGS__);
-        #endif
-    #else
-        // XXX shameless hack for clang around variadic printf...
-        //     Compilies w/o supplying -std=c++11 but shows warning,
-        //     so we sielence them :)
-        #pragma clang diagnostic ignored "-Wc++11-extensions"
-        #pragma clang diagnostic ignored "-Wunnamed-type-template-args"
-            template <class... Args>
-            inline __host__ __device__ void va_printf(char const* format, Args const&... args)
-            {
-        #ifdef __CUDA_ARCH__
-              printf(format, blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, args...);
-        #else
-              printf(format, args...);
-        #endif
-            }
-        #ifndef __CUDA_ARCH__
-            #define _CubLog(format, ...) va_printf(format,__VA_ARGS__);
-        #else
-            #define _CubLog(format, ...) va_printf("[block (%d,%d,%d), thread (%d,%d,%d)]: " format, __VA_ARGS__);
-        #endif
-    #endif
-#endif
-
-
-
-
-/** @} */       // end group UtilMgmt
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/util_device.cuh b/pfsimulator/third_party/cub/util_device.cuh
deleted file mode 100644
index a5f3b6144..000000000
--- a/pfsimulator/third_party/cub/util_device.cuh
+++ /dev/null
@@ -1,347 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Properties of a given CUDA device and the corresponding PTX bundle
- */
-
-#pragma once
-
-#include "util_type.cuh"
-#include "util_arch.cuh"
-#include "util_debug.cuh"
-#include "util_namespace.cuh"
-#include "util_macro.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \addtogroup UtilMgmt
- * @{
- */
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-
-/**
- * Alias temporaries to externally-allocated device storage (or simply return the amount of storage needed).
- */
-template <int ALLOCATIONS>
-__host__ __device__ __forceinline__
-cudaError_t AliasTemporaries(
-    void    *d_temp_storage,                    ///< [in] %Device-accessible allocation of temporary storage.  When NULL, the required allocation size is written to \p temp_storage_bytes and no work is done.
-    size_t  &temp_storage_bytes,                ///< [in,out] Size in bytes of \t d_temp_storage allocation
-    void*   (&allocations)[ALLOCATIONS],        ///< [in,out] Pointers to device allocations needed
-    size_t  (&allocation_sizes)[ALLOCATIONS])   ///< [in] Sizes in bytes of device allocations needed
-{
-    const int ALIGN_BYTES   = 256;
-    const int ALIGN_MASK    = ~(ALIGN_BYTES - 1);
-
-    // Compute exclusive prefix sum over allocation requests
-    size_t allocation_offsets[ALLOCATIONS];
-    size_t bytes_needed = 0;
-    for (int i = 0; i < ALLOCATIONS; ++i)
-    {
-        size_t allocation_bytes = (allocation_sizes[i] + ALIGN_BYTES - 1) & ALIGN_MASK;
-        allocation_offsets[i] = bytes_needed;
-        bytes_needed += allocation_bytes;
-    }
-    bytes_needed += ALIGN_BYTES - 1;
-
-    // Check if the caller is simply requesting the size of the storage allocation
-    if (!d_temp_storage)
-    {
-        temp_storage_bytes = bytes_needed;
-        return cudaSuccess;
-    }
-
-    // Check if enough storage provided
-    if (temp_storage_bytes < bytes_needed)
-    {
-        return CubDebug(cudaErrorInvalidValue);
-    }
-
-    // Alias
-    d_temp_storage = (void *) ((size_t(d_temp_storage) + ALIGN_BYTES - 1) & ALIGN_MASK);
-    for (int i = 0; i < ALLOCATIONS; ++i)
-    {
-        allocations[i] = static_cast<char*>(d_temp_storage) + allocation_offsets[i];
-    }
-
-    return cudaSuccess;
-}
-
-
-/**
- * Empty kernel for querying PTX manifest metadata (e.g., version) for the current device
- */
-template <typename T>
-__global__ void EmptyKernel(void) { }
-
-
-#endif  // DOXYGEN_SHOULD_SKIP_THIS
-
-/**
- * \brief Retrieves the PTX version that will be used on the current device (major * 100 + minor * 10)
- */
-CUB_RUNTIME_FUNCTION __forceinline__ cudaError_t PtxVersion(int &ptx_version)
-{
-    struct Dummy
-    {
-        /// Type definition of the EmptyKernel kernel entry point
-        typedef void (*EmptyKernelPtr)();
-
-        /// Force EmptyKernel<void> to be generated if this class is used
-        CUB_RUNTIME_FUNCTION __forceinline__
-        EmptyKernelPtr Empty()
-        {
-            return EmptyKernel<void>;
-        }
-    };
-
-
-#ifndef CUB_RUNTIME_ENABLED
-    (void)ptx_version;
-
-    // CUDA API calls not supported from this device
-    return cudaErrorInvalidConfiguration;
-
-#elif (CUB_PTX_ARCH > 0)
-
-    ptx_version = CUB_PTX_ARCH;
-    return cudaSuccess;
-
-#else
-
-    cudaError_t error = cudaSuccess;
-    do
-    {
-        cudaFuncAttributes empty_kernel_attrs;
-        if (CubDebug(error = cudaFuncGetAttributes(&empty_kernel_attrs, EmptyKernel<void>))) break;
-        ptx_version = empty_kernel_attrs.ptxVersion * 10;
-    }
-    while (0);
-
-    return error;
-
-#endif
-}
-
-
-/**
- * \brief Retrieves the SM version (major * 100 + minor * 10)
- */
-CUB_RUNTIME_FUNCTION __forceinline__ cudaError_t SmVersion(int &sm_version, int device_ordinal)
-{
-#ifndef CUB_RUNTIME_ENABLED
-    (void)sm_version;
-    (void)device_ordinal;
-
-    // CUDA API calls not supported from this device
-    return cudaErrorInvalidConfiguration;
-
-#else
-
-    cudaError_t error = cudaSuccess;
-    do
-    {
-        // Fill in SM version
-        int major, minor;
-        if (CubDebug(error = cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, device_ordinal))) break;
-        if (CubDebug(error = cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, device_ordinal))) break;
-        sm_version = major * 100 + minor * 10;
-    }
-    while (0);
-
-    return error;
-
-#endif
-}
-
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-/**
- * Synchronize the stream if specified
- */
-CUB_RUNTIME_FUNCTION __forceinline__
-static cudaError_t SyncStream(cudaStream_t stream)
-{
-#if (CUB_PTX_ARCH == 0)
-    return cudaStreamSynchronize(stream);
-#else
-    (void)stream;
-    // Device can't yet sync on a specific stream
-    return cudaDeviceSynchronize();
-#endif
-}
-
-
-/**
- * \brief Computes maximum SM occupancy in thread blocks for executing the given kernel function pointer \p kernel_ptr on the current device with \p block_threads per thread block.
- *
- * \par Snippet
- * The code snippet below illustrates the use of the MaxSmOccupancy function.
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/util_device.cuh>
- *
- * template <typename T>
- * __global__ void ExampleKernel()
- * {
- *     // Allocate shared memory for BlockScan
- *     __shared__ volatile T buffer[4096];
- *
- *        ...
- * }
- *
- *     ...
- *
- * // Determine SM occupancy for ExampleKernel specialized for unsigned char
- * int max_sm_occupancy;
- * MaxSmOccupancy(max_sm_occupancy, ExampleKernel<unsigned char>, 64);
- *
- * // max_sm_occupancy  <-- 4 on SM10
- * // max_sm_occupancy  <-- 8 on SM20
- * // max_sm_occupancy  <-- 12 on SM35
- *
- * \endcode
- *
- */
-template <typename KernelPtr>
-CUB_RUNTIME_FUNCTION __forceinline__
-cudaError_t MaxSmOccupancy(
-    int                 &max_sm_occupancy,          ///< [out] maximum number of thread blocks that can reside on a single SM
-    KernelPtr           kernel_ptr,                 ///< [in] Kernel pointer for which to compute SM occupancy
-    int                 block_threads,              ///< [in] Number of threads per thread block
-    int                 dynamic_smem_bytes = 0)
-{
-#ifndef CUB_RUNTIME_ENABLED
-    (void)dynamic_smem_bytes;
-    (void)block_threads;
-    (void)kernel_ptr;
-    (void)max_sm_occupancy;
-
-    // CUDA API calls not supported from this device
-    return CubDebug(cudaErrorInvalidConfiguration);
-
-#else
-
-    return cudaOccupancyMaxActiveBlocksPerMultiprocessor (
-        &max_sm_occupancy,
-        kernel_ptr,
-        block_threads,
-        dynamic_smem_bytes);
-
-#endif  // CUB_RUNTIME_ENABLED
-}
-
-
-/******************************************************************************
- * Policy management
- ******************************************************************************/
-
-/**
- * Kernel dispatch configuration
- */
-struct KernelConfig
-{
-    int block_threads;
-    int items_per_thread;
-    int tile_size;
-    int sm_occupancy;
-
-    CUB_RUNTIME_FUNCTION __forceinline__
-    KernelConfig() : block_threads(0), items_per_thread(0), tile_size(0), sm_occupancy(0) {}
-
-    template <typename AgentPolicyT, typename KernelPtrT>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    cudaError_t Init(KernelPtrT kernel_ptr)
-    {
-        block_threads        = AgentPolicyT::BLOCK_THREADS;
-        items_per_thread     = AgentPolicyT::ITEMS_PER_THREAD;
-        tile_size            = block_threads * items_per_thread;
-        cudaError_t retval   = MaxSmOccupancy(sm_occupancy, kernel_ptr, block_threads);
-        return retval;
-    }
-};
-
-
-
-/// Helper for dispatching into a policy chain
-template <int PTX_VERSION, typename PolicyT, typename PrevPolicyT>
-struct ChainedPolicy
-{
-   /// The policy for the active compiler pass
-   typedef typename If<(CUB_PTX_ARCH < PTX_VERSION), typename PrevPolicyT::ActivePolicy, PolicyT>::Type ActivePolicy;
-
-   /// Specializes and dispatches op in accordance to the first policy in the chain of adequate PTX version
-   template <typename FunctorT>
-   CUB_RUNTIME_FUNCTION __forceinline__
-   static cudaError_t Invoke(int ptx_version, FunctorT &op)
-   {
-       if (ptx_version < PTX_VERSION) {
-           return PrevPolicyT::Invoke(ptx_version, op);
-       }
-       return op.template Invoke<PolicyT>();
-   }
-};
-
-/// Helper for dispatching into a policy chain (end-of-chain specialization)
-template <int PTX_VERSION, typename PolicyT>
-struct ChainedPolicy<PTX_VERSION, PolicyT, PolicyT>
-{
-    /// The policy for the active compiler pass
-    typedef PolicyT ActivePolicy;
-
-    /// Specializes and dispatches op in accordance to the first policy in the chain of adequate PTX version
-    template <typename FunctorT>
-    CUB_RUNTIME_FUNCTION __forceinline__
-    static cudaError_t Invoke(int /*ptx_version*/, FunctorT &op) {
-        return op.template Invoke<PolicyT>();
-    }
-};
-
-
-
-
-#endif  // Do not document
-
-
-
-
-/** @} */       // end group UtilMgmt
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/util_macro.cuh b/pfsimulator/third_party/cub/util_macro.cuh
deleted file mode 100644
index ff8636542..000000000
--- a/pfsimulator/third_party/cub/util_macro.cuh
+++ /dev/null
@@ -1,103 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/******************************************************************************
- * Common C/C++ macro utilities
- ******************************************************************************/
-
-#pragma once
-
-#include "util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \addtogroup UtilModule
- * @{
- */
-
-#ifndef CUB_ALIGN
-    #if defined(_WIN32) || defined(_WIN64)
-        /// Align struct
-        #define CUB_ALIGN(bytes) __declspec(align(32))
-    #else
-        /// Align struct
-        #define CUB_ALIGN(bytes) __attribute__((aligned(bytes)))
-    #endif
-#endif
-
-#ifndef CUB_MAX
-    /// Select maximum(a, b)
-    #define CUB_MAX(a, b) (((b) > (a)) ? (b) : (a))
-#endif
-
-#ifndef CUB_MIN
-    /// Select minimum(a, b)
-    #define CUB_MIN(a, b) (((b) < (a)) ? (b) : (a))
-#endif
-
-#ifndef CUB_QUOTIENT_FLOOR
-    /// Quotient of x/y rounded down to nearest integer
-    #define CUB_QUOTIENT_FLOOR(x, y) ((x) / (y))
-#endif
-
-#ifndef CUB_QUOTIENT_CEILING
-    /// Quotient of x/y rounded up to nearest integer
-    #define CUB_QUOTIENT_CEILING(x, y) (((x) + (y) - 1) / (y))
-#endif
-
-#ifndef CUB_ROUND_UP_NEAREST
-    /// x rounded up to the nearest multiple of y
-    #define CUB_ROUND_UP_NEAREST(x, y) ((((x) + (y) - 1) / (y)) * y)
-#endif
-
-#ifndef CUB_ROUND_DOWN_NEAREST
-    /// x rounded down to the nearest multiple of y
-    #define CUB_ROUND_DOWN_NEAREST(x, y) (((x) / (y)) * y)
-#endif
-
-
-#ifndef CUB_STATIC_ASSERT
-    #ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-        #define CUB_CAT_(a, b) a ## b
-        #define CUB_CAT(a, b) CUB_CAT_(a, b)
-    #endif // DOXYGEN_SHOULD_SKIP_THIS
-
-    /// Static assert
-    #define CUB_STATIC_ASSERT(cond, msg) typedef int CUB_CAT(cub_static_assert, __LINE__)[(cond) ? 1 : -1]
-#endif
-
-/** @} */       // end group UtilModule
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/util_namespace.cuh b/pfsimulator/third_party/cub/util_namespace.cuh
deleted file mode 100644
index c3136229f..000000000
--- a/pfsimulator/third_party/cub/util_namespace.cuh
+++ /dev/null
@@ -1,50 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Place-holder for prefixing the cub namespace
- */
-
-#pragma once
-
-// For example:
-//#define CUB_NS_PREFIX namespace thrust{ namespace detail {
-//#define CUB_NS_POSTFIX } }
-
-#ifndef CUB_NS_PREFIX
-#define CUB_NS_PREFIX
-#endif
-
-#ifndef CUB_NS_POSTFIX
-#define CUB_NS_POSTFIX
-#endif
-
-#ifndef CUB_NS_QUALIFIER
-#define CUB_NS_QUALIFIER ::cub
-#endif
diff --git a/pfsimulator/third_party/cub/util_ptx.cuh b/pfsimulator/third_party/cub/util_ptx.cuh
deleted file mode 100644
index 582ca0d8b..000000000
--- a/pfsimulator/third_party/cub/util_ptx.cuh
+++ /dev/null
@@ -1,758 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * PTX intrinsics
- */
-
-
-#pragma once
-
-#include "util_type.cuh"
-#include "util_arch.cuh"
-#include "util_namespace.cuh"
-#include "util_debug.cuh"
-
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \addtogroup UtilPtx
- * @{
- */
-
-
-/******************************************************************************
- * PTX helper macros
- ******************************************************************************/
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-/**
- * Register modifier for pointer-types (for inlining PTX assembly)
- */
-#if defined(_WIN64) || defined(__LP64__)
-    #define __CUB_LP64__ 1
-    // 64-bit register modifier for inlined asm
-    #define _CUB_ASM_PTR_ "l"
-    #define _CUB_ASM_PTR_SIZE_ "u64"
-#else
-    #define __CUB_LP64__ 0
-    // 32-bit register modifier for inlined asm
-    #define _CUB_ASM_PTR_ "r"
-    #define _CUB_ASM_PTR_SIZE_ "u32"
-#endif
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-/******************************************************************************
- * Inlined PTX intrinsics
- ******************************************************************************/
-
-/**
- * \brief Shift-right then add.  Returns (\p x >> \p shift) + \p addend.
- */
-__device__ __forceinline__ unsigned int SHR_ADD(
-    unsigned int x,
-    unsigned int shift,
-    unsigned int addend)
-{
-    unsigned int ret;
-#if CUB_PTX_ARCH >= 200
-    asm ("vshr.u32.u32.u32.clamp.add %0, %1, %2, %3;" :
-        "=r"(ret) : "r"(x), "r"(shift), "r"(addend));
-#else
-    ret = (x >> shift) + addend;
-#endif
-    return ret;
-}
-
-
-/**
- * \brief Shift-left then add.  Returns (\p x << \p shift) + \p addend.
- */
-__device__ __forceinline__ unsigned int SHL_ADD(
-    unsigned int x,
-    unsigned int shift,
-    unsigned int addend)
-{
-    unsigned int ret;
-#if CUB_PTX_ARCH >= 200
-    asm ("vshl.u32.u32.u32.clamp.add %0, %1, %2, %3;" :
-        "=r"(ret) : "r"(x), "r"(shift), "r"(addend));
-#else
-    ret = (x << shift) + addend;
-#endif
-    return ret;
-}
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-/**
- * Bitfield-extract.
- */
-template <typename UnsignedBits, int BYTE_LEN>
-__device__ __forceinline__ unsigned int BFE(
-    UnsignedBits            source,
-    unsigned int            bit_start,
-    unsigned int            num_bits,
-    Int2Type<BYTE_LEN>      /*byte_len*/)
-{
-    unsigned int bits;
-#if CUB_PTX_ARCH >= 200
-    asm ("bfe.u32 %0, %1, %2, %3;" : "=r"(bits) : "r"((unsigned int) source), "r"(bit_start), "r"(num_bits));
-#else
-    const unsigned int MASK = (1 << num_bits) - 1;
-    bits = (source >> bit_start) & MASK;
-#endif
-    return bits;
-}
-
-
-/**
- * Bitfield-extract for 64-bit types.
- */
-template <typename UnsignedBits>
-__device__ __forceinline__ unsigned int BFE(
-    UnsignedBits            source,
-    unsigned int            bit_start,
-    unsigned int            num_bits,
-    Int2Type<8>             /*byte_len*/)
-{
-    const unsigned long long MASK = (1ull << num_bits) - 1;
-    return (source >> bit_start) & MASK;
-}
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-/**
- * \brief Bitfield-extract.  Extracts \p num_bits from \p source starting at bit-offset \p bit_start.  The input \p source may be an 8b, 16b, 32b, or 64b unsigned integer type.
- */
-template <typename UnsignedBits>
-__device__ __forceinline__ unsigned int BFE(
-    UnsignedBits source,
-    unsigned int bit_start,
-    unsigned int num_bits)
-{
-    return BFE(source, bit_start, num_bits, Int2Type<sizeof(UnsignedBits)>());
-}
-
-
-/**
- * \brief Bitfield insert.  Inserts the \p num_bits least significant bits of \p y into \p x at bit-offset \p bit_start.
- */
-__device__ __forceinline__ void BFI(
-    unsigned int &ret,
-    unsigned int x,
-    unsigned int y,
-    unsigned int bit_start,
-    unsigned int num_bits)
-{
-#if CUB_PTX_ARCH >= 200
-    asm ("bfi.b32 %0, %1, %2, %3, %4;" :
-        "=r"(ret) : "r"(y), "r"(x), "r"(bit_start), "r"(num_bits));
-#else
-    x <<= bit_start;
-    unsigned int MASK_X = ((1 << num_bits) - 1) << bit_start;
-    unsigned int MASK_Y = ~MASK_X;
-    ret = (y & MASK_Y) | (x & MASK_X);
-#endif
-}
-
-
-/**
- * \brief Three-operand add.  Returns \p x + \p y + \p z.
- */
-__device__ __forceinline__ unsigned int IADD3(unsigned int x, unsigned int y, unsigned int z)
-{
-#if CUB_PTX_ARCH >= 200
-    asm ("vadd.u32.u32.u32.add %0, %1, %2, %3;" : "=r"(x) : "r"(x), "r"(y), "r"(z));
-#else
-    x = x + y + z;
-#endif
-    return x;
-}
-
-
-/**
- * \brief Byte-permute. Pick four arbitrary bytes from two 32-bit registers, and reassemble them into a 32-bit destination register.  For SM2.0 or later.
- *
- * \par
- * The bytes in the two source registers \p a and \p b are numbered from 0 to 7:
- * {\p b, \p a} = {{b7, b6, b5, b4}, {b3, b2, b1, b0}}. For each of the four bytes
- * {b3, b2, b1, b0} selected in the return value, a 4-bit selector is defined within
- * the four lower "nibbles" of \p index: {\p index } = {n7, n6, n5, n4, n3, n2, n1, n0}
- *
- * \par Snippet
- * The code snippet below illustrates byte-permute.
- * \par
- * \code
- * #include <cub/cub.cuh>
- *
- * __global__ void ExampleKernel(...)
- * {
- *     int a        = 0x03020100;
- *     int b        = 0x07060504;
- *     int index    = 0x00007531;
- *
- *     int selected = PRMT(a, b, index);    // 0x07050301
- *
- * \endcode
- *
- */
-__device__ __forceinline__ int PRMT(unsigned int a, unsigned int b, unsigned int index)
-{
-    int ret;
-    asm ("prmt.b32 %0, %1, %2, %3;" : "=r"(ret) : "r"(a), "r"(b), "r"(index));
-    return ret;
-}
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-/**
- * Sync-threads barrier.
- */
-__device__ __forceinline__ void BAR(int count)
-{
-    asm volatile("bar.sync 1, %0;" : : "r"(count));
-}
-
-/**
- * CTA barrier
- */
-__device__  __forceinline__ void CTA_SYNC()
-{
-    __syncthreads();
-}
-
-
-/**
- * CTA barrier with predicate
- */
-__device__  __forceinline__ int CTA_SYNC_AND(int p)
-{
-    return __syncthreads_and(p);
-}
-
-
-/**
- * Warp barrier
- */
-__device__  __forceinline__ void WARP_SYNC(unsigned int member_mask)
-{
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-    __syncwarp(member_mask);
-#endif
-}
-
-
-/**
- * Warp any
- */
-__device__  __forceinline__ int WARP_ANY(int predicate, unsigned int member_mask)
-{
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-    return __any_sync(member_mask, predicate);
-#else
-    return ::__any(predicate);
-#endif
-}
-
-
-/**
- * Warp any
- */
-__device__  __forceinline__ int WARP_ALL(int predicate, unsigned int member_mask)
-{
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-    return __all_sync(member_mask, predicate);
-#else
-    return ::__all(predicate);
-#endif
-}
-
-
-/**
- * Warp ballot
- */
-__device__  __forceinline__ int WARP_BALLOT(int predicate, unsigned int member_mask)
-{
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-    return __ballot_sync(member_mask, predicate);
-#else
-    return __ballot(predicate);
-#endif
-}
-
-/**
- * Warp synchronous shfl_up
- */
-__device__ __forceinline__ 
-unsigned int SHFL_UP_SYNC(unsigned int word, int src_offset, int flags, unsigned int member_mask)
-{
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-    asm volatile("shfl.sync.up.b32 %0, %1, %2, %3, %4;"
-        : "=r"(word) : "r"(word), "r"(src_offset), "r"(flags), "r"(member_mask));
-#else
-    asm volatile("shfl.up.b32 %0, %1, %2, %3;"
-        : "=r"(word) : "r"(word), "r"(src_offset), "r"(flags));
-#endif
-    return word;
-}
-
-/**
- * Warp synchronous shfl_down
- */
-__device__ __forceinline__ 
-unsigned int SHFL_DOWN_SYNC(unsigned int word, int src_offset, int flags, unsigned int member_mask)
-{
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-    asm volatile("shfl.sync.down.b32 %0, %1, %2, %3, %4;"
-        : "=r"(word) : "r"(word), "r"(src_offset), "r"(flags), "r"(member_mask));
-#else
-    asm volatile("shfl.down.b32 %0, %1, %2, %3;"
-        : "=r"(word) : "r"(word), "r"(src_offset), "r"(flags));
-#endif
-    return word;
-}
-
-/**
- * Warp synchronous shfl_idx
- */
-__device__ __forceinline__ 
-unsigned int SHFL_IDX_SYNC(unsigned int word, int src_lane, int flags, unsigned int member_mask)
-{
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-    asm volatile("shfl.sync.idx.b32 %0, %1, %2, %3, %4;"
-        : "=r"(word) : "r"(word), "r"(src_lane), "r"(flags), "r"(member_mask));
-#else
-    asm volatile("shfl.idx.b32 %0, %1, %2, %3;"
-        : "=r"(word) : "r"(word), "r"(src_lane), "r"(flags));
-#endif
-    return word;
-}
-
-/**
- * Floating point multiply. (Mantissa LSB rounds towards zero.)
- */
-__device__ __forceinline__ float FMUL_RZ(float a, float b)
-{
-    float d;
-    asm ("mul.rz.f32 %0, %1, %2;" : "=f"(d) : "f"(a), "f"(b));
-    return d;
-}
-
-
-/**
- * Floating point multiply-add. (Mantissa LSB rounds towards zero.)
- */
-__device__ __forceinline__ float FFMA_RZ(float a, float b, float c)
-{
-    float d;
-    asm ("fma.rz.f32 %0, %1, %2, %3;" : "=f"(d) : "f"(a), "f"(b), "f"(c));
-    return d;
-}
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-/**
- * \brief Terminates the calling thread
- */
-__device__ __forceinline__ void ThreadExit() {
-    asm volatile("exit;");
-}    
-
-
-/**
- * \brief  Abort execution and generate an interrupt to the host CPU
- */
-__device__ __forceinline__ void ThreadTrap() {
-    asm volatile("trap;");
-}
-
-
-/**
- * \brief Returns the row-major linear thread identifier for a multidimensional thread block
- */
-__device__ __forceinline__ int RowMajorTid(int block_dim_x, int block_dim_y, int block_dim_z)
-{
-    return ((block_dim_z == 1) ? 0 : (threadIdx.z * block_dim_x * block_dim_y)) +
-            ((block_dim_y == 1) ? 0 : (threadIdx.y * block_dim_x)) +
-            threadIdx.x;
-}
-
-
-/**
- * \brief Returns the warp lane ID of the calling thread
- */
-__device__ __forceinline__ unsigned int LaneId()
-{
-    unsigned int ret;
-    asm ("mov.u32 %0, %%laneid;" : "=r"(ret) );
-    return ret;
-}
-
-
-/**
- * \brief Returns the warp ID of the calling thread.  Warp ID is guaranteed to be unique among warps, but may not correspond to a zero-based ranking within the thread block.
- */
-__device__ __forceinline__ unsigned int WarpId()
-{
-    unsigned int ret;
-    asm ("mov.u32 %0, %%warpid;" : "=r"(ret) );
-    return ret;
-}
-
-/**
- * \brief Returns the warp lane mask of all lanes less than the calling thread
- */
-__device__ __forceinline__ unsigned int LaneMaskLt()
-{
-    unsigned int ret;
-    asm ("mov.u32 %0, %%lanemask_lt;" : "=r"(ret) );
-    return ret;
-}
-
-/**
- * \brief Returns the warp lane mask of all lanes less than or equal to the calling thread
- */
-__device__ __forceinline__ unsigned int LaneMaskLe()
-{
-    unsigned int ret;
-    asm ("mov.u32 %0, %%lanemask_le;" : "=r"(ret) );
-    return ret;
-}
-
-/**
- * \brief Returns the warp lane mask of all lanes greater than the calling thread
- */
-__device__ __forceinline__ unsigned int LaneMaskGt()
-{
-    unsigned int ret;
-    asm ("mov.u32 %0, %%lanemask_gt;" : "=r"(ret) );
-    return ret;
-}
-
-/**
- * \brief Returns the warp lane mask of all lanes greater than or equal to the calling thread
- */
-__device__ __forceinline__ unsigned int LaneMaskGe()
-{
-    unsigned int ret;
-    asm ("mov.u32 %0, %%lanemask_ge;" : "=r"(ret) );
-    return ret;
-}
-
-/** @} */       // end group UtilPtx
-
-
-
-
-/**
- * \brief Shuffle-up for any data type.  Each <em>warp-lane<sub>i</sub></em> obtains the value \p input contributed by <em>warp-lane</em><sub><em>i</em>-<tt>src_offset</tt></sub>.  For thread lanes \e i < src_offset, the thread's own \p input is returned to the thread. ![](shfl_up_logo.png)
- * \ingroup WarpModule
- *
- * \tparam LOGICAL_WARP_THREADS     The number of threads per "logical" warp.  Must be a power-of-two <= 32.
- * \tparam T                        <b>[inferred]</b> The input/output element type
- *
- * \par
- * - Available only for SM3.0 or newer
- *
- * \par Snippet
- * The code snippet below illustrates each thread obtaining a \p double value from the
- * predecessor of its predecessor.
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/util_ptx.cuh>
- *
- * __global__ void ExampleKernel(...)
- * {
- *     // Obtain one input item per thread
- *     double thread_data = ...
- *
- *     // Obtain item from two ranks below
- *     double peer_data = ShuffleUp<32>(thread_data, 2, 0, 0xffffffff);
- *
- * \endcode
- * \par
- * Suppose the set of input \p thread_data across the first warp of threads is <tt>{1.0, 2.0, 3.0, 4.0, 5.0, ..., 32.0}</tt>.
- * The corresponding output \p peer_data will be <tt>{1.0, 2.0, 1.0, 2.0, 3.0, ..., 30.0}</tt>.
- *
- */
-template <
-    int LOGICAL_WARP_THREADS,   ///< Number of threads per logical warp
-    typename T>
-__device__ __forceinline__ T ShuffleUp(
-    T               input,              ///< [in] The value to broadcast
-    int             src_offset,         ///< [in] The relative down-offset of the peer to read from
-    int             first_thread,       ///< [in] Index of first lane in logical warp (typically 0)
-    unsigned int    member_mask)        ///< [in] 32-bit mask of participating warp lanes
-{
-    /// The 5-bit SHFL mask for logically splitting warps into sub-segments starts 8-bits up
-    enum {
-        SHFL_C = (32 - LOGICAL_WARP_THREADS) << 8
-    };
-
-    typedef typename UnitWord<T>::ShuffleWord ShuffleWord;
-
-    const int       WORDS           = (sizeof(T) + sizeof(ShuffleWord) - 1) / sizeof(ShuffleWord);
- 
-    T               output;
-    ShuffleWord     *output_alias   = reinterpret_cast<ShuffleWord *>(&output);
-    ShuffleWord     *input_alias    = reinterpret_cast<ShuffleWord *>(&input);
-
-    unsigned int shuffle_word;
-    shuffle_word = SHFL_UP_SYNC((unsigned int)input_alias[0], src_offset, first_thread | SHFL_C, member_mask);
-    output_alias[0] = shuffle_word;
-
-    #pragma unroll
-    for (int WORD = 1; WORD < WORDS; ++WORD)
-    {
-        shuffle_word       = SHFL_UP_SYNC((unsigned int)input_alias[WORD], src_offset, first_thread | SHFL_C, member_mask);
-        output_alias[WORD] = shuffle_word;
-    }
-
-    return output;
-}
-
-
-/**
- * \brief Shuffle-down for any data type.  Each <em>warp-lane<sub>i</sub></em> obtains the value \p input contributed by <em>warp-lane</em><sub><em>i</em>+<tt>src_offset</tt></sub>.  For thread lanes \e i >= WARP_THREADS, the thread's own \p input is returned to the thread.  ![](shfl_down_logo.png)
- * \ingroup WarpModule
- *
- * \tparam LOGICAL_WARP_THREADS     The number of threads per "logical" warp.  Must be a power-of-two <= 32.
- * \tparam T                        <b>[inferred]</b> The input/output element type
- *
- * \par
- * - Available only for SM3.0 or newer
- *
- * \par Snippet
- * The code snippet below illustrates each thread obtaining a \p double value from the
- * successor of its successor.
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/util_ptx.cuh>
- *
- * __global__ void ExampleKernel(...)
- * {
- *     // Obtain one input item per thread
- *     double thread_data = ...
- *
- *     // Obtain item from two ranks below
- *     double peer_data = ShuffleDown<32>(thread_data, 2, 31, 0xffffffff);
- *
- * \endcode
- * \par
- * Suppose the set of input \p thread_data across the first warp of threads is <tt>{1.0, 2.0, 3.0, 4.0, 5.0, ..., 32.0}</tt>.
- * The corresponding output \p peer_data will be <tt>{3.0, 4.0, 5.0, 6.0, 7.0, ..., 32.0}</tt>.
- *
- */
-template <
-    int LOGICAL_WARP_THREADS,   ///< Number of threads per logical warp
-    typename T>
-__device__ __forceinline__ T ShuffleDown(
-    T               input,              ///< [in] The value to broadcast
-    int             src_offset,         ///< [in] The relative up-offset of the peer to read from
-    int             last_thread,        ///< [in] Index of last thread in logical warp (typically 31 for a 32-thread warp)
-    unsigned int    member_mask)        ///< [in] 32-bit mask of participating warp lanes
-{
-    /// The 5-bit SHFL mask for logically splitting warps into sub-segments starts 8-bits up
-    enum {
-        SHFL_C = (32 - LOGICAL_WARP_THREADS) << 8
-    };
-
-    typedef typename UnitWord<T>::ShuffleWord ShuffleWord;
-
-    const int       WORDS           = (sizeof(T) + sizeof(ShuffleWord) - 1) / sizeof(ShuffleWord);
-
-    T               output;
-    ShuffleWord     *output_alias   = reinterpret_cast<ShuffleWord *>(&output);
-    ShuffleWord     *input_alias    = reinterpret_cast<ShuffleWord *>(&input);
-
-    unsigned int shuffle_word;
-    shuffle_word    = SHFL_DOWN_SYNC((unsigned int)input_alias[0], src_offset, last_thread | SHFL_C, member_mask);
-    output_alias[0] = shuffle_word;
-
-    #pragma unroll
-    for (int WORD = 1; WORD < WORDS; ++WORD)
-    {
-        shuffle_word       = SHFL_DOWN_SYNC((unsigned int)input_alias[WORD], src_offset, last_thread | SHFL_C, member_mask);
-        output_alias[WORD] = shuffle_word;
-    }
-
-    return output;
-}
-
-
-/**
- * \brief Shuffle-broadcast for any data type.  Each <em>warp-lane<sub>i</sub></em> obtains the value \p input
- * contributed by <em>warp-lane</em><sub><tt>src_lane</tt></sub>.  For \p src_lane < 0 or \p src_lane >= WARP_THREADS,
- * then the thread's own \p input is returned to the thread. ![](shfl_broadcast_logo.png)
- *
- * \tparam LOGICAL_WARP_THREADS     The number of threads per "logical" warp.  Must be a power-of-two <= 32.
- * \tparam T                        <b>[inferred]</b> The input/output element type
- *
- * \ingroup WarpModule
- *
- * \par
- * - Available only for SM3.0 or newer
- *
- * \par Snippet
- * The code snippet below illustrates each thread obtaining a \p double value from <em>warp-lane</em><sub>0</sub>.
- *
- * \par
- * \code
- * #include <cub/cub.cuh>   // or equivalently <cub/util_ptx.cuh>
- *
- * __global__ void ExampleKernel(...)
- * {
- *     // Obtain one input item per thread
- *     double thread_data = ...
- *
- *     // Obtain item from thread 0
- *     double peer_data = ShuffleIndex<32>(thread_data, 0, 0xffffffff);
- *
- * \endcode
- * \par
- * Suppose the set of input \p thread_data across the first warp of threads is <tt>{1.0, 2.0, 3.0, 4.0, 5.0, ..., 32.0}</tt>.
- * The corresponding output \p peer_data will be <tt>{1.0, 1.0, 1.0, 1.0, 1.0, ..., 1.0}</tt>.
- *
- */
-template <
-    int LOGICAL_WARP_THREADS,   ///< Number of threads per logical warp
-    typename T>
-__device__ __forceinline__ T ShuffleIndex(
-    T               input,                  ///< [in] The value to broadcast
-    int             src_lane,               ///< [in] Which warp lane is to do the broadcasting
-    unsigned int    member_mask)            ///< [in] 32-bit mask of participating warp lanes
-{
-    /// The 5-bit SHFL mask for logically splitting warps into sub-segments starts 8-bits up
-    enum {
-        SHFL_C = ((32 - LOGICAL_WARP_THREADS) << 8) | (LOGICAL_WARP_THREADS - 1)
-    };
-
-    typedef typename UnitWord<T>::ShuffleWord ShuffleWord;
-
-    const int       WORDS           = (sizeof(T) + sizeof(ShuffleWord) - 1) / sizeof(ShuffleWord);
-
-    T               output;
-    ShuffleWord     *output_alias   = reinterpret_cast<ShuffleWord *>(&output);
-    ShuffleWord     *input_alias    = reinterpret_cast<ShuffleWord *>(&input);
-
-    unsigned int shuffle_word;
-    shuffle_word = SHFL_IDX_SYNC((unsigned int)input_alias[0],
-                                 src_lane,
-                                 SHFL_C,
-                                 member_mask);
-
-    output_alias[0] = shuffle_word;
-
-    #pragma unroll
-    for (int WORD = 1; WORD < WORDS; ++WORD)
-    {
-        shuffle_word = SHFL_IDX_SYNC((unsigned int)input_alias[WORD],
-                                     src_lane,
-                                     SHFL_C,
-                                     member_mask);
-
-        output_alias[WORD] = shuffle_word;
-    }
-
-    return output;
-}
-
-
-
-/**
- * Compute a 32b mask of threads having the same least-significant
- * LABEL_BITS of \p label as the calling thread.
- */
-template <int LABEL_BITS>
-inline __device__ unsigned int MatchAny(unsigned int label)
-{
-    unsigned int retval;
-
-    // Extract masks of common threads for each bit
-    #pragma unroll
-    for (int BIT = 0; BIT < LABEL_BITS; ++BIT)
-    {
-        unsigned int mask;
-        unsigned int current_bit = 1 << BIT;
-        asm ("{\n"
-            "    .reg .pred p;\n"
-            "    and.b32 %0, %1, %2;"
-            "    setp.eq.u32 p, %0, %2;\n"
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-            "    vote.ballot.sync.b32 %0, p, 0xffffffff;\n"
-#else
-            "    vote.ballot.b32 %0, p;\n"
-#endif
-            "    @!p not.b32 %0, %0;\n"
-            "}\n" : "=r"(mask) : "r"(label), "r"(current_bit));
-
-        // Remove peers who differ
-        retval = (BIT == 0) ? mask : retval & mask;
-    }
-
-    return retval;
-
-//  // VOLTA match
-//    unsigned int retval;
-//    asm ("{\n"
-//         "    match.any.sync.b32 %0, %1, 0xffffffff;\n"
-//         "}\n" : "=r"(retval) : "r"(label));
-//    return retval;
-
-}
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/util_type.cuh b/pfsimulator/third_party/cub/util_type.cuh
deleted file mode 100644
index 0ba41e1ed..000000000
--- a/pfsimulator/third_party/cub/util_type.cuh
+++ /dev/null
@@ -1,1167 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * Common type manipulation (metaprogramming) utilities
- */
-
-#pragma once
-
-#include <iostream>
-#include <limits>
-#include <cfloat>
-
-#if (__CUDACC_VER_MAJOR__ >= 9)
-    #include <cuda_fp16.h>
-#endif
-
-#include "util_macro.cuh"
-#include "util_arch.cuh"
-#include "util_namespace.cuh"
-
-
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \addtogroup UtilModule
- * @{
- */
-
-
-
-/******************************************************************************
- * Type equality
- ******************************************************************************/
-
-/**
- * \brief Type selection (<tt>IF ? ThenType : ElseType</tt>)
- */
-template <bool IF, typename ThenType, typename ElseType>
-struct If
-{
-    /// Conditional type result
-    typedef ThenType Type;      // true
-};
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-template <typename ThenType, typename ElseType>
-struct If<false, ThenType, ElseType>
-{
-    typedef ElseType Type;      // false
-};
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-
-/******************************************************************************
- * Conditional types
- ******************************************************************************/
-
-/**
- * \brief Type equality test
- */
-template <typename A, typename B>
-struct Equals
-{
-    enum {
-        VALUE = 0,
-        NEGATE = 1
-    };
-};
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-template <typename A>
-struct Equals <A, A>
-{
-    enum {
-        VALUE = 1,
-        NEGATE = 0
-    };
-};
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-/******************************************************************************
- * Static math
- ******************************************************************************/
-
-/**
- * \brief Statically determine log2(N), rounded up.
- *
- * For example:
- *     Log2<8>::VALUE   // 3
- *     Log2<3>::VALUE   // 2
- */
-template <int N, int CURRENT_VAL = N, int COUNT = 0>
-struct Log2
-{
-    /// Static logarithm value
-    enum { VALUE = Log2<N, (CURRENT_VAL >> 1), COUNT + 1>::VALUE };         // Inductive case
-};
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-template <int N, int COUNT>
-struct Log2<N, 0, COUNT>
-{
-    enum {VALUE = (1 << (COUNT - 1) < N) ?                                  // Base case
-        COUNT :
-        COUNT - 1 };
-};
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-/**
- * \brief Statically determine if N is a power-of-two
- */
-template <int N>
-struct PowerOfTwo
-{
-    enum { VALUE = ((N & (N - 1)) == 0) };
-};
-
-
-
-/******************************************************************************
- * Pointer vs. iterator detection
- ******************************************************************************/
-
-/**
- * \brief Pointer vs. iterator
- */
-template <typename Tp>
-struct IsPointer
-{
-    enum { VALUE = 0 };
-};
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-template <typename Tp>
-struct IsPointer<Tp*>
-{
-    enum { VALUE = 1 };
-};
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-
-/******************************************************************************
- * Qualifier detection
- ******************************************************************************/
-
-/**
- * \brief Volatile modifier test
- */
-template <typename Tp>
-struct IsVolatile
-{
-    enum { VALUE = 0 };
-};
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-template <typename Tp>
-struct IsVolatile<Tp volatile>
-{
-    enum { VALUE = 1 };
-};
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-/******************************************************************************
- * Qualifier removal
- ******************************************************************************/
-
-/**
- * \brief Removes \p const and \p volatile qualifiers from type \p Tp.
- *
- * For example:
- *     <tt>typename RemoveQualifiers<volatile int>::Type         // int;</tt>
- */
-template <typename Tp, typename Up = Tp>
-struct RemoveQualifiers
-{
-    /// Type without \p const and \p volatile qualifiers
-    typedef Up Type;
-};
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-template <typename Tp, typename Up>
-struct RemoveQualifiers<Tp, volatile Up>
-{
-    typedef Up Type;
-};
-
-template <typename Tp, typename Up>
-struct RemoveQualifiers<Tp, const Up>
-{
-    typedef Up Type;
-};
-
-template <typename Tp, typename Up>
-struct RemoveQualifiers<Tp, const volatile Up>
-{
-    typedef Up Type;
-};
-
-
-/******************************************************************************
- * Marker types
- ******************************************************************************/
-
-/**
- * \brief A simple "NULL" marker type
- */
-struct NullType
-{
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-    template <typename T>
-    __host__ __device__ __forceinline__ NullType& operator =(const T&) { return *this; }
-
-    __host__ __device__ __forceinline__ bool operator ==(const NullType&) { return true; }
-
-    __host__ __device__ __forceinline__ bool operator !=(const NullType&) { return false; }
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-};
-
-
-/**
- * \brief Allows for the treatment of an integral constant as a type at compile-time (e.g., to achieve static call dispatch based on constant integral values)
- */
-template <int A>
-struct Int2Type
-{
-   enum {VALUE = A};
-};
-
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-
-/******************************************************************************
- * Size and alignment
- ******************************************************************************/
-
-/// Structure alignment
-template <typename T>
-struct AlignBytes
-{
-    struct Pad
-    {
-        T       val;
-        char    byte;
-    };
-
-    enum
-    {
-        /// The "true CUDA" alignment of T in bytes
-        ALIGN_BYTES = sizeof(Pad) - sizeof(T)
-    };
-
-    /// The "truly aligned" type
-    typedef T Type;
-};
-
-// Specializations where host C++ compilers (e.g., 32-bit Windows) may disagree
-// with device C++ compilers (EDG) on types passed as template parameters through
-// kernel functions
-
-#define __CUB_ALIGN_BYTES(t, b)         \
-    template <> struct AlignBytes<t>    \
-    { enum { ALIGN_BYTES = b }; typedef __align__(b) t Type; };
-
-__CUB_ALIGN_BYTES(short4, 8)
-__CUB_ALIGN_BYTES(ushort4, 8)
-__CUB_ALIGN_BYTES(int2, 8)
-__CUB_ALIGN_BYTES(uint2, 8)
-__CUB_ALIGN_BYTES(long long, 8)
-__CUB_ALIGN_BYTES(unsigned long long, 8)
-__CUB_ALIGN_BYTES(float2, 8)
-__CUB_ALIGN_BYTES(double, 8)
-#ifdef _WIN32
-    __CUB_ALIGN_BYTES(long2, 8)
-    __CUB_ALIGN_BYTES(ulong2, 8)
-#else
-    __CUB_ALIGN_BYTES(long2, 16)
-    __CUB_ALIGN_BYTES(ulong2, 16)
-#endif
-__CUB_ALIGN_BYTES(int4, 16)
-__CUB_ALIGN_BYTES(uint4, 16)
-__CUB_ALIGN_BYTES(float4, 16)
-__CUB_ALIGN_BYTES(long4, 16)
-__CUB_ALIGN_BYTES(ulong4, 16)
-__CUB_ALIGN_BYTES(longlong2, 16)
-__CUB_ALIGN_BYTES(ulonglong2, 16)
-__CUB_ALIGN_BYTES(double2, 16)
-__CUB_ALIGN_BYTES(longlong4, 16)
-__CUB_ALIGN_BYTES(ulonglong4, 16)
-__CUB_ALIGN_BYTES(double4, 16)
-
-template <typename T> struct AlignBytes<volatile T> : AlignBytes<T> {};
-template <typename T> struct AlignBytes<const T> : AlignBytes<T> {};
-template <typename T> struct AlignBytes<const volatile T> : AlignBytes<T> {};
-
-
-/// Unit-words of data movement
-template <typename T>
-struct UnitWord
-{
-    enum {
-        ALIGN_BYTES = AlignBytes<T>::ALIGN_BYTES
-    };
-
-    template <typename Unit>
-    struct IsMultiple
-    {
-        enum {
-            UNIT_ALIGN_BYTES    = AlignBytes<Unit>::ALIGN_BYTES,
-            IS_MULTIPLE         = (sizeof(T) % sizeof(Unit) == 0) && (ALIGN_BYTES % UNIT_ALIGN_BYTES == 0)
-        };
-    };
-
-    /// Biggest shuffle word that T is a whole multiple of and is not larger than the alignment of T
-    typedef typename If<IsMultiple<int>::IS_MULTIPLE,
-        unsigned int,
-        typename If<IsMultiple<short>::IS_MULTIPLE,
-            unsigned short,
-            unsigned char>::Type>::Type         ShuffleWord;
-
-    /// Biggest volatile word that T is a whole multiple of and is not larger than the alignment of T
-    typedef typename If<IsMultiple<long long>::IS_MULTIPLE,
-        unsigned long long,
-        ShuffleWord>::Type                      VolatileWord;
-
-    /// Biggest memory-access word that T is a whole multiple of and is not larger than the alignment of T
-    typedef typename If<IsMultiple<longlong2>::IS_MULTIPLE,
-        ulonglong2,
-        VolatileWord>::Type                     DeviceWord;
-
-    /// Biggest texture reference word that T is a whole multiple of and is not larger than the alignment of T
-    typedef typename If<IsMultiple<int4>::IS_MULTIPLE,
-        uint4,
-        typename If<IsMultiple<int2>::IS_MULTIPLE,
-            uint2,
-            ShuffleWord>::Type>::Type           TextureWord;
-};
-
-
-// float2 specialization workaround (for SM10-SM13)
-template <>
-struct UnitWord <float2>
-{
-    typedef int         ShuffleWord;
-#if (CUB_PTX_ARCH > 0) && (CUB_PTX_ARCH <= 130)
-    typedef float       VolatileWord;
-    typedef uint2       DeviceWord;
-#else
-    typedef unsigned long long   VolatileWord;
-    typedef unsigned long long   DeviceWord;
-#endif
-    typedef float2      TextureWord;
-};
-
-// float4 specialization workaround (for SM10-SM13)
-template <>
-struct UnitWord <float4>
-{
-    typedef int         ShuffleWord;
-#if (CUB_PTX_ARCH > 0) && (CUB_PTX_ARCH <= 130)
-    typedef float               VolatileWord;
-    typedef uint4               DeviceWord;
-#else
-    typedef unsigned long long  VolatileWord;
-    typedef ulonglong2          DeviceWord;
-#endif
-    typedef float4              TextureWord;
-};
-
-
-// char2 specialization workaround (for SM10-SM13)
-template <>
-struct UnitWord <char2>
-{
-    typedef unsigned short      ShuffleWord;
-#if (CUB_PTX_ARCH > 0) && (CUB_PTX_ARCH <= 130)
-    typedef unsigned short      VolatileWord;
-    typedef short               DeviceWord;
-#else
-    typedef unsigned short      VolatileWord;
-    typedef unsigned short      DeviceWord;
-#endif
-    typedef unsigned short      TextureWord;
-};
-
-
-template <typename T> struct UnitWord<volatile T> : UnitWord<T> {};
-template <typename T> struct UnitWord<const T> : UnitWord<T> {};
-template <typename T> struct UnitWord<const volatile T> : UnitWord<T> {};
-
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-
-/******************************************************************************
- * Vector type inference utilities.
- ******************************************************************************/
-
-/**
- * \brief Exposes a member typedef \p Type that names the corresponding CUDA vector type if one exists.  Otherwise \p Type refers to the CubVector structure itself, which will wrap the corresponding \p x, \p y, etc. vector fields.
- */
-template <typename T, int vec_elements> struct CubVector;
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-enum
-{
-    /// The maximum number of elements in CUDA vector types
-    MAX_VEC_ELEMENTS = 4,
-};
-
-
-/**
- * Generic vector-1 type
- */
-template <typename T>
-struct CubVector<T, 1>
-{
-    T x;
-
-    typedef T BaseType;
-    typedef CubVector<T, 1> Type;
-};
-
-/**
- * Generic vector-2 type
- */
-template <typename T>
-struct CubVector<T, 2>
-{
-    T x;
-    T y;
-
-    typedef T BaseType;
-    typedef CubVector<T, 2> Type;
-};
-
-/**
- * Generic vector-3 type
- */
-template <typename T>
-struct CubVector<T, 3>
-{
-    T x;
-    T y;
-    T z;
-
-    typedef T BaseType;
-    typedef CubVector<T, 3> Type;
-};
-
-/**
- * Generic vector-4 type
- */
-template <typename T>
-struct CubVector<T, 4>
-{
-    T x;
-    T y;
-    T z;
-    T w;
-
-    typedef T BaseType;
-    typedef CubVector<T, 4> Type;
-};
-
-
-/**
- * Macro for expanding partially-specialized built-in vector types
- */
-#define CUB_DEFINE_VECTOR_TYPE(base_type,short_type)                                                    \
-                                                                                                        \
-    template<> struct CubVector<base_type, 1> : short_type##1                                           \
-    {                                                                                                   \
-      typedef base_type       BaseType;                                                                 \
-      typedef short_type##1   Type;                                                                     \
-      __host__ __device__ __forceinline__ CubVector operator+(const CubVector &other) const {           \
-          CubVector retval;                                                                             \
-          retval.x = x + other.x;                                                                       \
-          return retval;                                                                                \
-      }                                                                                                 \
-      __host__ __device__ __forceinline__ CubVector operator-(const CubVector &other) const {           \
-          CubVector retval;                                                                             \
-          retval.x = x - other.x;                                                                       \
-          return retval;                                                                                \
-      }                                                                                                 \
-    };                                                                                                  \
-                                                                                                        \
-    template<> struct CubVector<base_type, 2> : short_type##2                                           \
-    {                                                                                                   \
-        typedef base_type       BaseType;                                                               \
-        typedef short_type##2   Type;                                                                   \
-        __host__ __device__ __forceinline__ CubVector operator+(const CubVector &other) const {         \
-            CubVector retval;                                                                           \
-            retval.x = x + other.x;                                                                     \
-            retval.y = y + other.y;                                                                     \
-            return retval;                                                                              \
-        }                                                                                               \
-        __host__ __device__ __forceinline__ CubVector operator-(const CubVector &other) const {         \
-            CubVector retval;                                                                           \
-            retval.x = x - other.x;                                                                     \
-            retval.y = y - other.y;                                                                     \
-            return retval;                                                                              \
-        }                                                                                               \
-    };                                                                                                  \
-                                                                                                        \
-    template<> struct CubVector<base_type, 3> : short_type##3                                           \
-    {                                                                                                   \
-        typedef base_type       BaseType;                                                               \
-        typedef short_type##3   Type;                                                                   \
-        __host__ __device__ __forceinline__ CubVector operator+(const CubVector &other) const {         \
-            CubVector retval;                                                                           \
-            retval.x = x + other.x;                                                                     \
-            retval.y = y + other.y;                                                                     \
-            retval.z = z + other.z;                                                                     \
-            return retval;                                                                              \
-        }                                                                                               \
-        __host__ __device__ __forceinline__ CubVector operator-(const CubVector &other) const {         \
-            CubVector retval;                                                                           \
-            retval.x = x - other.x;                                                                     \
-            retval.y = y - other.y;                                                                     \
-            retval.z = z - other.z;                                                                     \
-            return retval;                                                                              \
-        }                                                                                               \
-    };                                                                                                  \
-                                                                                                        \
-    template<> struct CubVector<base_type, 4> : short_type##4                                           \
-    {                                                                                                   \
-        typedef base_type       BaseType;                                                               \
-        typedef short_type##4   Type;                                                                   \
-        __host__ __device__ __forceinline__ CubVector operator+(const CubVector &other) const {         \
-            CubVector retval;                                                                           \
-            retval.x = x + other.x;                                                                     \
-            retval.y = y + other.y;                                                                     \
-            retval.z = z + other.z;                                                                     \
-            retval.w = w + other.w;                                                                     \
-            return retval;                                                                              \
-        }                                                                                               \
-        __host__ __device__ __forceinline__ CubVector operator-(const CubVector &other) const {         \
-            CubVector retval;                                                                           \
-            retval.x = x - other.x;                                                                     \
-            retval.y = y - other.y;                                                                     \
-            retval.z = z - other.z;                                                                     \
-            retval.w = w - other.w;                                                                     \
-            return retval;                                                                              \
-        }                                                                                               \
-    };
-
-
-
-// Expand CUDA vector types for built-in primitives
-CUB_DEFINE_VECTOR_TYPE(char,               char)
-CUB_DEFINE_VECTOR_TYPE(signed char,        char)
-CUB_DEFINE_VECTOR_TYPE(short,              short)
-CUB_DEFINE_VECTOR_TYPE(int,                int)
-CUB_DEFINE_VECTOR_TYPE(long,               long)
-CUB_DEFINE_VECTOR_TYPE(long long,          longlong)
-CUB_DEFINE_VECTOR_TYPE(unsigned char,      uchar)
-CUB_DEFINE_VECTOR_TYPE(unsigned short,     ushort)
-CUB_DEFINE_VECTOR_TYPE(unsigned int,       uint)
-CUB_DEFINE_VECTOR_TYPE(unsigned long,      ulong)
-CUB_DEFINE_VECTOR_TYPE(unsigned long long, ulonglong)
-CUB_DEFINE_VECTOR_TYPE(float,              float)
-CUB_DEFINE_VECTOR_TYPE(double,             double)
-CUB_DEFINE_VECTOR_TYPE(bool,               uchar)
-
-// Undefine macros
-#undef CUB_DEFINE_VECTOR_TYPE
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-
-/******************************************************************************
- * Wrapper types
- ******************************************************************************/
-
-/**
- * \brief A storage-backing wrapper that allows types with non-trivial constructors to be aliased in unions
- */
-template <typename T>
-struct Uninitialized
-{
-    /// Biggest memory-access word that T is a whole multiple of and is not larger than the alignment of T
-    typedef typename UnitWord<T>::DeviceWord DeviceWord;
-
-    enum
-    {
-        WORDS = sizeof(T) / sizeof(DeviceWord)
-    };
-
-    /// Backing storage
-    DeviceWord storage[WORDS];
-
-    /// Alias
-    __host__ __device__ __forceinline__ T& Alias()
-    {
-        return reinterpret_cast<T&>(*this);
-    }
-};
-
-
-/**
- * \brief A key identifier paired with a corresponding value
- */
-template <
-    typename    _Key,
-    typename    _Value
-#if defined(_WIN32) && !defined(_WIN64)
-    , bool KeyIsLT = (AlignBytes<_Key>::ALIGN_BYTES < AlignBytes<_Value>::ALIGN_BYTES)
-    , bool ValIsLT = (AlignBytes<_Value>::ALIGN_BYTES < AlignBytes<_Key>::ALIGN_BYTES)
-#endif // #if defined(_WIN32) && !defined(_WIN64)
-    >
-struct KeyValuePair
-{
-    typedef _Key    Key;                ///< Key data type
-    typedef _Value  Value;              ///< Value data type
-
-    Key     key;                        ///< Item key
-    Value   value;                      ///< Item value
-
-    /// Constructor
-    __host__ __device__ __forceinline__
-    KeyValuePair() {}
-
-    /// Constructor
-    __host__ __device__ __forceinline__
-    KeyValuePair(Key const& key, Value const& value) : key(key), value(value) {}
-
-    /// Inequality operator
-    __host__ __device__ __forceinline__ bool operator !=(const KeyValuePair &b)
-    {
-        return (value != b.value) || (key != b.key);
-    }
-};
-
-#if defined(_WIN32) && !defined(_WIN64)
-
-/**
- * Win32 won't do 16B alignment.  This can present two problems for
- * should-be-16B-aligned (but actually 8B aligned) built-in and intrinsics members:
- * 1) If a smaller-aligned item were to be listed first, the host compiler places the
- *    should-be-16B item at too early an offset (and disagrees with device compiler)
- * 2) Or, if a smaller-aligned item lists second, the host compiler gets the size
- *    of the struct wrong (and disagrees with device compiler)
- *
- * So we put the larger-should-be-aligned item first, and explicitly pad the
- * end of the struct
- */
-
-/// Smaller key specialization
-template <typename K, typename V>
-struct KeyValuePair<K, V, true, false>
-{
-    typedef K Key;
-    typedef V Value;
-
-    typedef char Pad[AlignBytes<V>::ALIGN_BYTES - AlignBytes<K>::ALIGN_BYTES];
-
-    Value   value;  // Value has larger would-be alignment and goes first
-    Key     key;
-    Pad     pad;
-
-    /// Constructor
-    __host__ __device__ __forceinline__
-    KeyValuePair() {}
-
-    /// Constructor
-    __host__ __device__ __forceinline__
-    KeyValuePair(Key const& key, Value const& value) : key(key), value(value) {}
-
-    /// Inequality operator
-    __host__ __device__ __forceinline__ bool operator !=(const KeyValuePair &b)
-    {
-        return (value != b.value) || (key != b.key);
-    }
-};
-
-
-/// Smaller value specialization
-template <typename K, typename V>
-struct KeyValuePair<K, V, false, true>
-{
-    typedef K Key;
-    typedef V Value;
-
-    typedef char Pad[AlignBytes<K>::ALIGN_BYTES - AlignBytes<V>::ALIGN_BYTES];
-
-    Key     key;    // Key has larger would-be alignment and goes first
-    Value   value;
-    Pad     pad;
-
-    /// Constructor
-    __host__ __device__ __forceinline__
-    KeyValuePair() {}
-
-    /// Constructor
-    __host__ __device__ __forceinline__
-    KeyValuePair(Key const& key, Value const& value) : key(key), value(value) {}
-
-    /// Inequality operator
-    __host__ __device__ __forceinline__ bool operator !=(const KeyValuePair &b)
-    {
-        return (value != b.value) || (key != b.key);
-    }
-};
-
-#endif // #if defined(_WIN32) && !defined(_WIN64)
-
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-
-/**
- * \brief A wrapper for passing simple static arrays as kernel parameters
- */
-template <typename T, int COUNT>
-struct ArrayWrapper
-{
-
-    /// Statically-sized array of type \p T
-    T array[COUNT];
-
-    /// Constructor
-    __host__ __device__ __forceinline__ ArrayWrapper() {}
-};
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-/**
- * \brief Double-buffer storage wrapper for multi-pass stream transformations that require more than one storage array for streaming intermediate results back and forth.
- *
- * Many multi-pass computations require a pair of "ping-pong" storage
- * buffers (e.g., one for reading from and the other for writing to, and then
- * vice-versa for the subsequent pass).  This structure wraps a set of device
- * buffers and a "selector" member to track which is "current".
- */
-template <typename T>
-struct DoubleBuffer
-{
-    /// Pair of device buffer pointers
-    T *d_buffers[2];
-
-    ///  Selector into \p d_buffers (i.e., the active/valid buffer)
-    int selector;
-
-    /// \brief Constructor
-    __host__ __device__ __forceinline__ DoubleBuffer()
-    {
-        selector = 0;
-        d_buffers[0] = NULL;
-        d_buffers[1] = NULL;
-    }
-
-    /// \brief Constructor
-    __host__ __device__ __forceinline__ DoubleBuffer(
-        T *d_current,         ///< The currently valid buffer
-        T *d_alternate)       ///< Alternate storage buffer of the same size as \p d_current
-    {
-        selector = 0;
-        d_buffers[0] = d_current;
-        d_buffers[1] = d_alternate;
-    }
-
-    /// \brief Return pointer to the currently valid buffer
-    __host__ __device__ __forceinline__ T* Current() { return d_buffers[selector]; }
-
-    /// \brief Return pointer to the currently invalid buffer
-    __host__ __device__ __forceinline__ T* Alternate() { return d_buffers[selector ^ 1]; }
-
-};
-
-
-
-/******************************************************************************
- * Typedef-detection
- ******************************************************************************/
-
-
-/**
- * \brief Defines a structure \p detector_name that is templated on type \p T.  The \p detector_name struct exposes a constant member \p VALUE indicating whether or not parameter \p T exposes a nested type \p nested_type_name
- */
-#define CUB_DEFINE_DETECT_NESTED_TYPE(detector_name, nested_type_name)  \
-    template <typename T>                                               \
-    struct detector_name                                                \
-    {                                                                   \
-        template <typename C>                                           \
-        static char& test(typename C::nested_type_name*);               \
-        template <typename>                                             \
-        static int& test(...);                                          \
-        enum                                                            \
-        {                                                               \
-            VALUE = sizeof(test<T>(0)) < sizeof(int)                    \
-        };                                                              \
-    };
-
-
-
-/******************************************************************************
- * Simple enable-if (similar to Boost)
- ******************************************************************************/
-
-/**
- * \brief Simple enable-if (similar to Boost)
- */
-template <bool Condition, class T = void>
-struct EnableIf
-{
-    /// Enable-if type for SFINAE dummy variables
-    typedef T Type;
-};
-
-
-template <class T>
-struct EnableIf<false, T> {};
-
-
-
-/******************************************************************************
- * Typedef-detection
- ******************************************************************************/
-
-/**
- * \brief Determine whether or not BinaryOp's functor is of the form <tt>bool operator()(const T& a, const T&b)</tt> or <tt>bool operator()(const T& a, const T&b, unsigned int idx)</tt>
- */
-template <typename T, typename BinaryOp>
-struct BinaryOpHasIdxParam
-{
-private:
-/*
-    template <typename BinaryOpT, bool (BinaryOpT::*)(const T &a, const T &b, unsigned int idx) const>  struct SFINAE1 {};
-    template <typename BinaryOpT, bool (BinaryOpT::*)(const T &a, const T &b, unsigned int idx)>        struct SFINAE2 {};
-    template <typename BinaryOpT, bool (BinaryOpT::*)(T a, T b, unsigned int idx) const>                struct SFINAE3 {};
-    template <typename BinaryOpT, bool (BinaryOpT::*)(T a, T b, unsigned int idx)>                      struct SFINAE4 {};
-*/
-    template <typename BinaryOpT, bool (BinaryOpT::*)(const T &a, const T &b, int idx) const>           struct SFINAE5 {};
-    template <typename BinaryOpT, bool (BinaryOpT::*)(const T &a, const T &b, int idx)>                 struct SFINAE6 {};
-    template <typename BinaryOpT, bool (BinaryOpT::*)(T a, T b, int idx) const>                         struct SFINAE7 {};
-    template <typename BinaryOpT, bool (BinaryOpT::*)(T a, T b, int idx)>                               struct SFINAE8 {};
-/*
-    template <typename BinaryOpT> static char Test(SFINAE1<BinaryOpT, &BinaryOpT::operator()> *);
-    template <typename BinaryOpT> static char Test(SFINAE2<BinaryOpT, &BinaryOpT::operator()> *);
-    template <typename BinaryOpT> static char Test(SFINAE3<BinaryOpT, &BinaryOpT::operator()> *);
-    template <typename BinaryOpT> static char Test(SFINAE4<BinaryOpT, &BinaryOpT::operator()> *);
-*/
-    template <typename BinaryOpT> __host__ __device__ static char Test(SFINAE5<BinaryOpT, &BinaryOpT::operator()> *);
-    template <typename BinaryOpT> __host__ __device__ static char Test(SFINAE6<BinaryOpT, &BinaryOpT::operator()> *);
-    template <typename BinaryOpT> __host__ __device__ static char Test(SFINAE7<BinaryOpT, &BinaryOpT::operator()> *);
-    template <typename BinaryOpT> __host__ __device__ static char Test(SFINAE8<BinaryOpT, &BinaryOpT::operator()> *);
-
-    template <typename BinaryOpT> static int Test(...);
-
-public:
-
-    /// Whether the functor BinaryOp has a third <tt>unsigned int</tt> index param
-    static const bool HAS_PARAM = sizeof(Test<BinaryOp>(NULL)) == sizeof(char);
-};
-
-
-
-
-/******************************************************************************
- * Simple type traits utilities.
- *
- * For example:
- *     Traits<int>::CATEGORY             // SIGNED_INTEGER
- *     Traits<NullType>::NULL_TYPE       // true
- *     Traits<uint4>::CATEGORY           // NOT_A_NUMBER
- *     Traits<uint4>::PRIMITIVE;         // false
- *
- ******************************************************************************/
-
-/**
- * \brief Basic type traits categories
- */
-enum Category
-{
-    NOT_A_NUMBER,
-    SIGNED_INTEGER,
-    UNSIGNED_INTEGER,
-    FLOATING_POINT
-};
-
-
-/**
- * \brief Basic type traits
- */
-template <Category _CATEGORY, bool _PRIMITIVE, bool _NULL_TYPE, typename _UnsignedBits, typename T>
-struct BaseTraits
-{
-    /// Category
-    static const Category CATEGORY      = _CATEGORY;
-    enum
-    {
-        PRIMITIVE       = _PRIMITIVE,
-        NULL_TYPE       = _NULL_TYPE,
-    };
-};
-
-
-/**
- * Basic type traits (unsigned primitive specialization)
- */
-template <typename _UnsignedBits, typename T>
-struct BaseTraits<UNSIGNED_INTEGER, true, false, _UnsignedBits, T>
-{
-    typedef _UnsignedBits       UnsignedBits;
-
-    static const Category       CATEGORY    = UNSIGNED_INTEGER;
-    static const UnsignedBits   LOWEST_KEY  = UnsignedBits(0);
-    static const UnsignedBits   MAX_KEY     = UnsignedBits(-1);
-
-    enum
-    {
-        PRIMITIVE       = true,
-        NULL_TYPE       = false,
-    };
-
-
-    static __device__ __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key)
-    {
-        return key;
-    }
-
-    static __device__ __forceinline__ UnsignedBits TwiddleOut(UnsignedBits key)
-    {
-        return key;
-    }
-
-    static __host__ __device__ __forceinline__ T Max()
-    {
-        UnsignedBits retval = MAX_KEY;
-        return reinterpret_cast<T&>(retval);
-    }
-
-    static __host__ __device__ __forceinline__ T Lowest()
-    {
-        UnsignedBits retval = LOWEST_KEY;
-        return reinterpret_cast<T&>(retval);
-    }
-};
-
-
-/**
- * Basic type traits (signed primitive specialization)
- */
-template <typename _UnsignedBits, typename T>
-struct BaseTraits<SIGNED_INTEGER, true, false, _UnsignedBits, T>
-{
-    typedef _UnsignedBits       UnsignedBits;
-
-    static const Category       CATEGORY    = SIGNED_INTEGER;
-    static const UnsignedBits   HIGH_BIT    = UnsignedBits(1) << ((sizeof(UnsignedBits) * 8) - 1);
-    static const UnsignedBits   LOWEST_KEY  = HIGH_BIT;
-    static const UnsignedBits   MAX_KEY     = UnsignedBits(-1) ^ HIGH_BIT;
-
-    enum
-    {
-        PRIMITIVE       = true,
-        NULL_TYPE       = false,
-    };
-
-    static __device__ __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key)
-    {
-        return key ^ HIGH_BIT;
-    };
-
-    static __device__ __forceinline__ UnsignedBits TwiddleOut(UnsignedBits key)
-    {
-        return key ^ HIGH_BIT;
-    };
-
-    static __host__ __device__ __forceinline__ T Max()
-    {
-        UnsignedBits retval = MAX_KEY;
-        return reinterpret_cast<T&>(retval);
-    }
-
-    static __host__ __device__ __forceinline__ T Lowest()
-    {
-        UnsignedBits retval = LOWEST_KEY;
-        return reinterpret_cast<T&>(retval);
-    }
-};
-
-template <typename _T>
-struct FpLimits;
-
-template <>
-struct FpLimits<float>
-{
-    static __host__ __device__ __forceinline__ float Max() {
-        return FLT_MAX;
-    }
-
-    static __host__ __device__ __forceinline__ float Lowest() {
-        return FLT_MAX * float(-1);
-    }
-};
-
-template <>
-struct FpLimits<double>
-{
-    static __host__ __device__ __forceinline__ double Max() {
-        return DBL_MAX;
-    }
-
-    static __host__ __device__ __forceinline__ double Lowest() {
-        return DBL_MAX  * double(-1);
-    }
-};
-
-
-#if (__CUDACC_VER_MAJOR__ >= 9)
-template <>
-struct FpLimits<__half>
-{
-    static __host__ __device__ __forceinline__ __half Max() {
-        unsigned short max_word = 0x7BFF;
-        return reinterpret_cast<__half&>(max_word);
-    }
-
-    static __host__ __device__ __forceinline__ __half Lowest() {
-        unsigned short lowest_word = 0xFBFF;
-        return reinterpret_cast<__half&>(lowest_word);
-    }
-};
-#endif
-
-
-/**
- * Basic type traits (fp primitive specialization)
- */
-template <typename _UnsignedBits, typename T>
-struct BaseTraits<FLOATING_POINT, true, false, _UnsignedBits, T>
-{
-    typedef _UnsignedBits       UnsignedBits;
-
-    static const Category       CATEGORY    = FLOATING_POINT;
-    static const UnsignedBits   HIGH_BIT    = UnsignedBits(1) << ((sizeof(UnsignedBits) * 8) - 1);
-    static const UnsignedBits   LOWEST_KEY  = UnsignedBits(-1);
-    static const UnsignedBits   MAX_KEY     = UnsignedBits(-1) ^ HIGH_BIT;
-
-    enum
-    {
-        PRIMITIVE       = true,
-        NULL_TYPE       = false,
-    };
-
-    static __device__ __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key)
-    {
-        UnsignedBits mask = (key & HIGH_BIT) ? UnsignedBits(-1) : HIGH_BIT;
-        return key ^ mask;
-    };
-
-    static __device__ __forceinline__ UnsignedBits TwiddleOut(UnsignedBits key)
-    {
-        UnsignedBits mask = (key & HIGH_BIT) ? HIGH_BIT : UnsignedBits(-1);
-        return key ^ mask;
-    };
-
-    static __host__ __device__ __forceinline__ T Max() {
-        return FpLimits<T>::Max();
-    }
-
-    static __host__ __device__ __forceinline__ T Lowest() {
-        return FpLimits<T>::Lowest();
-    }
-};
-
-
-/**
- * \brief Numeric type traits
- */
-template <typename T> struct NumericTraits :            BaseTraits<NOT_A_NUMBER, false, false, T, T> {};
-
-template <> struct NumericTraits<NullType> :            BaseTraits<NOT_A_NUMBER, false, true, NullType, NullType> {};
-
-template <> struct NumericTraits<char> :                BaseTraits<(std::numeric_limits<char>::is_signed) ? SIGNED_INTEGER : UNSIGNED_INTEGER, true, false, unsigned char, char> {};
-template <> struct NumericTraits<signed char> :         BaseTraits<SIGNED_INTEGER, true, false, unsigned char, signed char> {};
-template <> struct NumericTraits<short> :               BaseTraits<SIGNED_INTEGER, true, false, unsigned short, short> {};
-template <> struct NumericTraits<int> :                 BaseTraits<SIGNED_INTEGER, true, false, unsigned int, int> {};
-template <> struct NumericTraits<long> :                BaseTraits<SIGNED_INTEGER, true, false, unsigned long, long> {};
-template <> struct NumericTraits<long long> :           BaseTraits<SIGNED_INTEGER, true, false, unsigned long long, long long> {};
-
-template <> struct NumericTraits<unsigned char> :       BaseTraits<UNSIGNED_INTEGER, true, false, unsigned char, unsigned char> {};
-template <> struct NumericTraits<unsigned short> :      BaseTraits<UNSIGNED_INTEGER, true, false, unsigned short, unsigned short> {};
-template <> struct NumericTraits<unsigned int> :        BaseTraits<UNSIGNED_INTEGER, true, false, unsigned int, unsigned int> {};
-template <> struct NumericTraits<unsigned long> :       BaseTraits<UNSIGNED_INTEGER, true, false, unsigned long, unsigned long> {};
-template <> struct NumericTraits<unsigned long long> :  BaseTraits<UNSIGNED_INTEGER, true, false, unsigned long long, unsigned long long> {};
-
-template <> struct NumericTraits<float> :               BaseTraits<FLOATING_POINT, true, false, unsigned int, float> {};
-template <> struct NumericTraits<double> :              BaseTraits<FLOATING_POINT, true, false, unsigned long long, double> {};
-#if (__CUDACC_VER_MAJOR__ >= 9)
-    template <> struct NumericTraits<__half> :          BaseTraits<FLOATING_POINT, true, false, unsigned short, __half> {};
-#endif
-
-template <> struct NumericTraits<bool> :                BaseTraits<UNSIGNED_INTEGER, true, false, typename UnitWord<bool>::VolatileWord, bool> {};
-
-
-
-/**
- * \brief Type traits
- */
-template <typename T>
-struct Traits : NumericTraits<typename RemoveQualifiers<T>::Type> {};
-
-
-#endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-/** @} */       // end group UtilModule
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/warp/specializations/warp_reduce_shfl.cuh b/pfsimulator/third_party/cub/warp/specializations/warp_reduce_shfl.cuh
deleted file mode 100644
index bbbf37e5c..000000000
--- a/pfsimulator/third_party/cub/warp/specializations/warp_reduce_shfl.cuh
+++ /dev/null
@@ -1,541 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::WarpReduceShfl provides SHFL-based variants of parallel reduction of items partitioned across a CUDA thread warp.
- */
-
-#pragma once
-
-#include "../../thread/thread_operators.cuh"
-#include "../../util_ptx.cuh"
-#include "../../util_type.cuh"
-#include "../../util_macro.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \brief WarpReduceShfl provides SHFL-based variants of parallel reduction of items partitioned across a CUDA thread warp.
- *
- * LOGICAL_WARP_THREADS must be a power-of-two
- */
-template <
-    typename    T,                      ///< Data type being reduced
-    int         LOGICAL_WARP_THREADS,   ///< Number of threads per logical warp
-    int         PTX_ARCH>               ///< The PTX compute capability for which to to specialize this collective
-struct WarpReduceShfl
-{
-    //---------------------------------------------------------------------
-    // Constants and type definitions
-    //---------------------------------------------------------------------
-
-    enum
-    {
-        /// Whether the logical warp size and the PTX warp size coincide
-        IS_ARCH_WARP = (LOGICAL_WARP_THREADS == CUB_WARP_THREADS(PTX_ARCH)),
-
-        /// The number of warp reduction steps
-        STEPS = Log2<LOGICAL_WARP_THREADS>::VALUE,
-
-        /// Number of logical warps in a PTX warp
-        LOGICAL_WARPS = CUB_WARP_THREADS(PTX_ARCH) / LOGICAL_WARP_THREADS,
-
-        /// The 5-bit SHFL mask for logically splitting warps into sub-segments starts 8-bits up
-        SHFL_C = (CUB_WARP_THREADS(PTX_ARCH) - LOGICAL_WARP_THREADS) << 8
-
-    };
-
-    template <typename S>
-    struct IsInteger
-    {
-        enum {
-            ///Whether the data type is a small (32b or less) integer for which we can use a single SFHL instruction per exchange
-            IS_SMALL_UNSIGNED = (Traits<S>::CATEGORY == UNSIGNED_INTEGER) && (sizeof(S) <= sizeof(unsigned int))
-        };
-    };
-
-
-    /// Shared memory storage layout type
-    typedef NullType TempStorage;
-
-
-    //---------------------------------------------------------------------
-    // Thread fields
-    //---------------------------------------------------------------------
-
-    /// Lane index in logical warp
-    unsigned int lane_id;
-
-    /// Logical warp index in 32-thread physical warp
-    unsigned int warp_id;
-
-    /// 32-thread physical warp member mask of logical warp
-    unsigned int member_mask;
-
-
-    //---------------------------------------------------------------------
-    // Construction
-    //---------------------------------------------------------------------
-
-    /// Constructor
-    __device__ __forceinline__ WarpReduceShfl(
-        TempStorage &/*temp_storage*/)
-    {
-        lane_id = LaneId();
-        warp_id = 0;
-        member_mask = 0xffffffffu >> (CUB_WARP_THREADS(PTX_ARCH) - LOGICAL_WARP_THREADS);
-
-        if (!IS_ARCH_WARP)
-        {
-            warp_id = lane_id / LOGICAL_WARP_THREADS;
-            lane_id = lane_id % LOGICAL_WARP_THREADS;
-            member_mask = member_mask << (warp_id * LOGICAL_WARP_THREADS);
-        }
-    }
-
-
-    //---------------------------------------------------------------------
-    // Reduction steps
-    //---------------------------------------------------------------------
-
-    /// Reduction (specialized for summation across uint32 types)
-    __device__ __forceinline__ unsigned int ReduceStep(
-        unsigned int    input,              ///< [in] Calling thread's input item.
-        cub::Sum        /*reduction_op*/,   ///< [in] Binary reduction operator
-        int             last_lane,          ///< [in] Index of last lane in segment
-        int             offset)             ///< [in] Up-offset to pull from
-    {
-        unsigned int output;
-        int shfl_c = last_lane | SHFL_C;   // Shuffle control (mask and last_lane)
-
-        // Use predicate set from SHFL to guard against invalid peers
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-        asm volatile(
-            "{"
-            "  .reg .u32 r0;"
-            "  .reg .pred p;"
-            "  shfl.sync.down.b32 r0|p, %1, %2, %3, %5;"
-            "  @p add.u32 r0, r0, %4;"
-            "  mov.u32 %0, r0;"
-            "}"
-            : "=r"(output) : "r"(input), "r"(offset), "r"(shfl_c), "r"(input), "r"(member_mask));
-#else
-        asm volatile(
-            "{"
-            "  .reg .u32 r0;"
-            "  .reg .pred p;"
-            "  shfl.down.b32 r0|p, %1, %2, %3;"
-            "  @p add.u32 r0, r0, %4;"
-            "  mov.u32 %0, r0;"
-            "}"
-            : "=r"(output) : "r"(input), "r"(offset), "r"(shfl_c), "r"(input));
-#endif
-
-        return output;
-    }
-
-
-    /// Reduction (specialized for summation across fp32 types)
-    __device__ __forceinline__ float ReduceStep(
-        float           input,              ///< [in] Calling thread's input item.
-        cub::Sum        /*reduction_op*/,   ///< [in] Binary reduction operator
-        int             last_lane,          ///< [in] Index of last lane in segment
-        int             offset)             ///< [in] Up-offset to pull from
-    {
-        float output;
-        int shfl_c = last_lane | SHFL_C;   // Shuffle control (mask and last_lane)
-
-        // Use predicate set from SHFL to guard against invalid peers
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-        asm volatile(
-            "{"
-            "  .reg .f32 r0;"
-            "  .reg .pred p;"
-            "  shfl.sync.down.b32 r0|p, %1, %2, %3, %5;"
-            "  @p add.f32 r0, r0, %4;"
-            "  mov.f32 %0, r0;"
-            "}"
-            : "=f"(output) : "f"(input), "r"(offset), "r"(shfl_c), "f"(input), "r"(member_mask));
-#else
-        asm volatile(
-            "{"
-            "  .reg .f32 r0;"
-            "  .reg .pred p;"
-            "  shfl.down.b32 r0|p, %1, %2, %3;"
-            "  @p add.f32 r0, r0, %4;"
-            "  mov.f32 %0, r0;"
-            "}"
-            : "=f"(output) : "f"(input), "r"(offset), "r"(shfl_c), "f"(input));
-#endif
-
-        return output;
-    }
-
-
-    /// Reduction (specialized for summation across unsigned long long types)
-    __device__ __forceinline__ unsigned long long ReduceStep(
-        unsigned long long  input,              ///< [in] Calling thread's input item.
-        cub::Sum            /*reduction_op*/,   ///< [in] Binary reduction operator
-        int                 last_lane,          ///< [in] Index of last lane in segment
-        int                 offset)             ///< [in] Up-offset to pull from
-    {
-        unsigned long long output;
-        int shfl_c = last_lane | SHFL_C;   // Shuffle control (mask and last_lane)
-
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-        asm volatile(
-            "{"
-            "  .reg .u32 lo;"
-            "  .reg .u32 hi;"
-            "  .reg .pred p;"
-            "  mov.b64 {lo, hi}, %1;"
-            "  shfl.sync.down.b32 lo|p, lo, %2, %3, %4;"
-            "  shfl.sync.down.b32 hi|p, hi, %2, %3, %4;"
-            "  mov.b64 %0, {lo, hi};"
-            "  @p add.u64 %0, %0, %1;"
-            "}"
-            : "=l"(output) : "l"(input), "r"(offset), "r"(shfl_c), "r"(member_mask));
-#else
-        asm volatile(
-            "{"
-            "  .reg .u32 lo;"
-            "  .reg .u32 hi;"
-            "  .reg .pred p;"
-            "  mov.b64 {lo, hi}, %1;"
-            "  shfl.down.b32 lo|p, lo, %2, %3;"
-            "  shfl.down.b32 hi|p, hi, %2, %3;"
-            "  mov.b64 %0, {lo, hi};"
-            "  @p add.u64 %0, %0, %1;"
-            "}"
-            : "=l"(output) : "l"(input), "r"(offset), "r"(shfl_c));
-#endif
-
-        return output;
-    }
-
-
-    /// Reduction (specialized for summation across long long types)
-    __device__ __forceinline__ long long ReduceStep(
-        long long           input,              ///< [in] Calling thread's input item.
-        cub::Sum            /*reduction_op*/,   ///< [in] Binary reduction operator
-        int                 last_lane,          ///< [in] Index of last lane in segment
-        int                 offset)             ///< [in] Up-offset to pull from
-    {
-        long long output;
-        int shfl_c = last_lane | SHFL_C;   // Shuffle control (mask and last_lane)
-
-        // Use predicate set from SHFL to guard against invalid peers
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-        asm volatile(
-            "{"
-            "  .reg .u32 lo;"
-            "  .reg .u32 hi;"
-            "  .reg .pred p;"
-            "  mov.b64 {lo, hi}, %1;"
-            "  shfl.sync.down.b32 lo|p, lo, %2, %3, %4;"
-            "  shfl.sync.down.b32 hi|p, hi, %2, %3, %4;"
-            "  mov.b64 %0, {lo, hi};"
-            "  @p add.s64 %0, %0, %1;"
-            "}"
-            : "=l"(output) : "l"(input), "r"(offset), "r"(shfl_c), "r"(member_mask));
-#else
-        asm volatile(
-            "{"
-            "  .reg .u32 lo;"
-            "  .reg .u32 hi;"
-            "  .reg .pred p;"
-            "  mov.b64 {lo, hi}, %1;"
-            "  shfl.down.b32 lo|p, lo, %2, %3;"
-            "  shfl.down.b32 hi|p, hi, %2, %3;"
-            "  mov.b64 %0, {lo, hi};"
-            "  @p add.s64 %0, %0, %1;"
-            "}"
-            : "=l"(output) : "l"(input), "r"(offset), "r"(shfl_c));
-#endif
-
-        return output;
-    }
-
-
-    /// Reduction (specialized for summation across double types)
-    __device__ __forceinline__ double ReduceStep(
-        double              input,              ///< [in] Calling thread's input item.
-        cub::Sum            /*reduction_op*/,   ///< [in] Binary reduction operator
-        int                 last_lane,          ///< [in] Index of last lane in segment
-        int                 offset)             ///< [in] Up-offset to pull from
-    {
-        double output;
-        int shfl_c = last_lane | SHFL_C;   // Shuffle control (mask and last_lane)
-
-        // Use predicate set from SHFL to guard against invalid peers
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-        asm volatile(
-            "{"
-            "  .reg .u32 lo;"
-            "  .reg .u32 hi;"
-            "  .reg .pred p;"
-            "  .reg .f64 r0;"
-            "  mov.b64 %0, %1;"
-            "  mov.b64 {lo, hi}, %1;"
-            "  shfl.sync.down.b32 lo|p, lo, %2, %3, %4;"
-            "  shfl.sync.down.b32 hi|p, hi, %2, %3, %4;"
-            "  mov.b64 r0, {lo, hi};"
-            "  @p add.f64 %0, %0, r0;"
-            "}"
-            : "=d"(output) : "d"(input), "r"(offset), "r"(shfl_c), "r"(member_mask));
-#else
-        asm volatile(
-            "{"
-            "  .reg .u32 lo;"
-            "  .reg .u32 hi;"
-            "  .reg .pred p;"
-            "  .reg .f64 r0;"
-            "  mov.b64 %0, %1;"
-            "  mov.b64 {lo, hi}, %1;"
-            "  shfl.down.b32 lo|p, lo, %2, %3;"
-            "  shfl.down.b32 hi|p, hi, %2, %3;"
-            "  mov.b64 r0, {lo, hi};"
-            "  @p add.f64 %0, %0, r0;"
-            "}"
-            : "=d"(output) : "d"(input), "r"(offset), "r"(shfl_c));
-#endif
-
-        return output;
-    }
-
-
-    /// Reduction (specialized for swizzled ReduceByKeyOp<cub::Sum> across KeyValuePair<KeyT, ValueT> types)
-    template <typename ValueT, typename KeyT>
-    __device__ __forceinline__ KeyValuePair<KeyT, ValueT> ReduceStep(
-        KeyValuePair<KeyT, ValueT>                  input,              ///< [in] Calling thread's input item.
-        SwizzleScanOp<ReduceByKeyOp<cub::Sum> >     /*reduction_op*/,   ///< [in] Binary reduction operator
-        int                                         last_lane,          ///< [in] Index of last lane in segment
-        int                                         offset)             ///< [in] Up-offset to pull from
-    {
-        KeyValuePair<KeyT, ValueT> output;
-
-        KeyT other_key = ShuffleDown<LOGICAL_WARP_THREADS>(input.key, offset, last_lane, member_mask);
-        
-        output.key = input.key;
-        output.value = ReduceStep(
-            input.value, 
-            cub::Sum(), 
-            last_lane, 
-            offset, 
-            Int2Type<IsInteger<ValueT>::IS_SMALL_UNSIGNED>());
-
-        if (input.key != other_key)
-            output.value = input.value;
-
-        return output;
-    }
-
-
-
-    /// Reduction (specialized for swizzled ReduceBySegmentOp<cub::Sum> across KeyValuePair<OffsetT, ValueT> types)
-    template <typename ValueT, typename OffsetT>
-    __device__ __forceinline__ KeyValuePair<OffsetT, ValueT> ReduceStep(
-        KeyValuePair<OffsetT, ValueT>                 input,              ///< [in] Calling thread's input item.
-        SwizzleScanOp<ReduceBySegmentOp<cub::Sum> >   /*reduction_op*/,   ///< [in] Binary reduction operator
-        int                                           last_lane,          ///< [in] Index of last lane in segment
-        int                                           offset)             ///< [in] Up-offset to pull from
-    {
-        KeyValuePair<OffsetT, ValueT> output;
-
-        output.value = ReduceStep(input.value, cub::Sum(), last_lane, offset, Int2Type<IsInteger<ValueT>::IS_SMALL_UNSIGNED>());
-        output.key = ReduceStep(input.key, cub::Sum(), last_lane, offset, Int2Type<IsInteger<OffsetT>::IS_SMALL_UNSIGNED>());
-
-        if (input.key > 0)
-            output.value = input.value;
-
-        return output;
-    }
-
-
-    /// Reduction step (generic)
-    template <typename _T, typename ReductionOp>
-    __device__ __forceinline__ _T ReduceStep(
-        _T                  input,              ///< [in] Calling thread's input item.
-        ReductionOp         reduction_op,       ///< [in] Binary reduction operator
-        int                 last_lane,          ///< [in] Index of last lane in segment
-        int                 offset)             ///< [in] Up-offset to pull from
-    {
-        _T output = input;
-
-        _T temp = ShuffleDown<LOGICAL_WARP_THREADS>(output, offset, last_lane, member_mask);
-
-        // Perform reduction op if valid
-        if (offset + lane_id <= last_lane)
-            output = reduction_op(input, temp);
-
-        return output;
-    }
-
-
-    /// Reduction step (specialized for small unsigned integers size 32b or less)
-    template <typename _T, typename ReductionOp>
-    __device__ __forceinline__ _T ReduceStep(
-        _T              input,                  ///< [in] Calling thread's input item.
-        ReductionOp     reduction_op,           ///< [in] Binary reduction operator
-        int             last_lane,              ///< [in] Index of last lane in segment
-        int             offset,                 ///< [in] Up-offset to pull from
-        Int2Type<true>  /*is_small_unsigned*/)  ///< [in] Marker type indicating whether T is a small unsigned integer
-    {
-        return ReduceStep(input, reduction_op, last_lane, offset);
-    }
-
-
-    /// Reduction step (specialized for types other than small unsigned integers size 32b or less)
-    template <typename _T, typename ReductionOp>
-    __device__ __forceinline__ _T ReduceStep(
-        _T              input,                  ///< [in] Calling thread's input item.
-        ReductionOp     reduction_op,           ///< [in] Binary reduction operator
-        int             last_lane,              ///< [in] Index of last lane in segment
-        int             offset,                 ///< [in] Up-offset to pull from
-        Int2Type<false> /*is_small_unsigned*/)  ///< [in] Marker type indicating whether T is a small unsigned integer
-    {
-        return ReduceStep(input, reduction_op, last_lane, offset);
-    }
-
-
-    //---------------------------------------------------------------------
-    // Templated inclusive scan iteration
-    //---------------------------------------------------------------------
-
-    template <typename ReductionOp, int STEP>
-    __device__ __forceinline__ void ReduceStep(
-        T&              input,              ///< [in] Calling thread's input item.
-        ReductionOp     reduction_op,       ///< [in] Binary reduction operator
-        int             last_lane,          ///< [in] Index of last lane in segment
-        Int2Type<STEP>  /*step*/)
-    {
-        input = ReduceStep(input, reduction_op, last_lane, 1 << STEP, Int2Type<IsInteger<T>::IS_SMALL_UNSIGNED>());
-
-        ReduceStep(input, reduction_op, last_lane, Int2Type<STEP + 1>());
-    }
-
-    template <typename ReductionOp>
-    __device__ __forceinline__ void ReduceStep(
-        T&              /*input*/,              ///< [in] Calling thread's input item.
-        ReductionOp     /*reduction_op*/,       ///< [in] Binary reduction operator
-        int             /*last_lane*/,          ///< [in] Index of last lane in segment
-        Int2Type<STEPS> /*step*/)
-    {}
-
-
-    //---------------------------------------------------------------------
-    // Reduction operations
-    //---------------------------------------------------------------------
-
-    /// Reduction
-    template <
-        bool            ALL_LANES_VALID,        ///< Whether all lanes in each warp are contributing a valid fold of items
-        typename        ReductionOp>
-    __device__ __forceinline__ T Reduce(
-        T               input,                  ///< [in] Calling thread's input
-        int             valid_items,            ///< [in] Total number of valid items across the logical warp
-        ReductionOp     reduction_op)           ///< [in] Binary reduction operator
-    {
-        int last_lane = (ALL_LANES_VALID) ?
-                            LOGICAL_WARP_THREADS - 1 :
-                            valid_items - 1;
-
-        T output = input;
-
-//        // Iterate reduction steps
-//        #pragma unroll
-//        for (int STEP = 0; STEP < STEPS; STEP++)
-//        {
-//            output = ReduceStep(output, reduction_op, last_lane, 1 << STEP, Int2Type<IsInteger<T>::IS_SMALL_UNSIGNED>());
-//        }
-
-        // Template-iterate reduction steps
-        ReduceStep(output, reduction_op, last_lane, Int2Type<0>());
-
-        return output;
-    }
-
-
-    /// Segmented reduction
-    template <
-        bool            HEAD_SEGMENTED,     ///< Whether flags indicate a segment-head or a segment-tail
-        typename        FlagT,
-        typename        ReductionOp>
-    __device__ __forceinline__ T SegmentedReduce(
-        T               input,              ///< [in] Calling thread's input
-        FlagT           flag,               ///< [in] Whether or not the current lane is a segment head/tail
-        ReductionOp     reduction_op)       ///< [in] Binary reduction operator
-    {
-        // Get the start flags for each thread in the warp.
-        int warp_flags = WARP_BALLOT(flag, member_mask);
-
-        // Convert to tail-segmented
-        if (HEAD_SEGMENTED)
-            warp_flags >>= 1;
-
-        // Mask out the bits below the current thread
-        warp_flags &= LaneMaskGe();
-
-        // Mask of physical lanes outside the logical warp and convert to logical lanemask
-        if (!IS_ARCH_WARP)
-        {
-            warp_flags = (warp_flags & member_mask) >> (warp_id * LOGICAL_WARP_THREADS);
-        }
-
-        // Mask in the last lane of logical warp
-        warp_flags |= 1u << (LOGICAL_WARP_THREADS - 1);
-
-        // Find the next set flag
-        int last_lane = __clz(__brev(warp_flags));
-
-        T output = input;
-
-//        // Iterate reduction steps
-//        #pragma unroll
-//        for (int STEP = 0; STEP < STEPS; STEP++)
-//        {
-//            output = ReduceStep(output, reduction_op, last_lane, 1 << STEP, Int2Type<IsInteger<T>::IS_SMALL_UNSIGNED>());
-//        }
-
-        // Template-iterate reduction steps
-        ReduceStep(output, reduction_op, last_lane, Int2Type<0>());
-
-        return output;
-    }
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/warp/specializations/warp_reduce_smem.cuh b/pfsimulator/third_party/cub/warp/specializations/warp_reduce_smem.cuh
deleted file mode 100644
index 7baa573be..000000000
--- a/pfsimulator/third_party/cub/warp/specializations/warp_reduce_smem.cuh
+++ /dev/null
@@ -1,372 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::WarpReduceSmem provides smem-based variants of parallel reduction of items partitioned across a CUDA thread warp.
- */
-
-#pragma once
-
-#include "../../thread/thread_operators.cuh"
-#include "../../thread/thread_load.cuh"
-#include "../../thread/thread_store.cuh"
-#include "../../util_type.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \brief WarpReduceSmem provides smem-based variants of parallel reduction of items partitioned across a CUDA thread warp.
- */
-template <
-    typename    T,                      ///< Data type being reduced
-    int         LOGICAL_WARP_THREADS,   ///< Number of threads per logical warp
-    int         PTX_ARCH>               ///< The PTX compute capability for which to to specialize this collective
-struct WarpReduceSmem
-{
-    /******************************************************************************
-     * Constants and type definitions
-     ******************************************************************************/
-
-    enum
-    {
-        /// Whether the logical warp size and the PTX warp size coincide
-        IS_ARCH_WARP = (LOGICAL_WARP_THREADS == CUB_WARP_THREADS(PTX_ARCH)),
-
-        /// Whether the logical warp size is a power-of-two
-        IS_POW_OF_TWO = PowerOfTwo<LOGICAL_WARP_THREADS>::VALUE,
-
-        /// The number of warp scan steps
-        STEPS = Log2<LOGICAL_WARP_THREADS>::VALUE,
-
-        /// The number of threads in half a warp
-        HALF_WARP_THREADS = 1 << (STEPS - 1),
-
-        /// The number of shared memory elements per warp
-        WARP_SMEM_ELEMENTS =  LOGICAL_WARP_THREADS + HALF_WARP_THREADS,
-
-        /// FlagT status (when not using ballot)
-        UNSET   = 0x0,  // Is initially unset
-        SET     = 0x1,  // Is initially set
-        SEEN    = 0x2,  // Has seen another head flag from a successor peer
-    };
-
-    /// Shared memory flag type
-    typedef unsigned char SmemFlag;
-
-    /// Shared memory storage layout type (1.5 warps-worth of elements for each warp)
-    struct _TempStorage
-    {
-        T           reduce[WARP_SMEM_ELEMENTS];
-        SmemFlag    flags[WARP_SMEM_ELEMENTS];
-    };
-
-    // Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    /******************************************************************************
-     * Thread fields
-     ******************************************************************************/
-
-    _TempStorage    &temp_storage;
-    unsigned int    lane_id;
-    unsigned int    member_mask;
-
-
-    /******************************************************************************
-     * Construction
-     ******************************************************************************/
-
-    /// Constructor
-    __device__ __forceinline__ WarpReduceSmem(
-        TempStorage     &temp_storage)
-    :
-        temp_storage(temp_storage.Alias()),
-
-        lane_id(IS_ARCH_WARP ?
-            LaneId() :
-            LaneId() % LOGICAL_WARP_THREADS),
-
-        member_mask((0xffffffff >> (32 - LOGICAL_WARP_THREADS)) << ((IS_ARCH_WARP || !IS_POW_OF_TWO ) ?
-            0 : // arch-width and non-power-of-two subwarps cannot be tiled with the arch-warp
-            ((LaneId() / LOGICAL_WARP_THREADS) * LOGICAL_WARP_THREADS)))
-    {}
-
-    /******************************************************************************
-     * Utility methods
-     ******************************************************************************/
-
-    //---------------------------------------------------------------------
-    // Regular reduction
-    //---------------------------------------------------------------------
-
-    /**
-     * Reduction step
-     */
-    template <
-        bool                ALL_LANES_VALID,        ///< Whether all lanes in each warp are contributing a valid fold of items
-        typename            ReductionOp,
-        int                 STEP>
-    __device__ __forceinline__ T ReduceStep(
-        T                   input,                  ///< [in] Calling thread's input
-        int                 valid_items,            ///< [in] Total number of valid items across the logical warp
-        ReductionOp         reduction_op,           ///< [in] Reduction operator
-        Int2Type<STEP>      /*step*/)
-    {
-        const int OFFSET = 1 << STEP;
-
-        // Share input through buffer
-        ThreadStore<STORE_VOLATILE>(&temp_storage.reduce[lane_id], input);
-
-        WARP_SYNC(member_mask);
-
-        // Update input if peer_addend is in range
-        if ((ALL_LANES_VALID && IS_POW_OF_TWO) || ((lane_id + OFFSET) < valid_items))
-        {
-            T peer_addend = ThreadLoad<LOAD_VOLATILE>(&temp_storage.reduce[lane_id + OFFSET]);
-            input = reduction_op(input, peer_addend);
-        }
-
-        WARP_SYNC(member_mask);
-
-        return ReduceStep<ALL_LANES_VALID>(input, valid_items, reduction_op, Int2Type<STEP + 1>());
-    }
-
-
-    /**
-     * Reduction step (terminate)
-     */
-    template <
-        bool                ALL_LANES_VALID,            ///< Whether all lanes in each warp are contributing a valid fold of items
-        typename            ReductionOp>
-    __device__ __forceinline__ T ReduceStep(
-        T                   input,                      ///< [in] Calling thread's input
-        int                 valid_items,                ///< [in] Total number of valid items across the logical warp
-        ReductionOp         /*reduction_op*/,           ///< [in] Reduction operator
-        Int2Type<STEPS>     /*step*/)
-    {
-        return input;
-    }
-
-
-    //---------------------------------------------------------------------
-    // Segmented reduction
-    //---------------------------------------------------------------------
-
-
-    /**
-     * Ballot-based segmented reduce
-     */
-    template <
-        bool            HEAD_SEGMENTED,     ///< Whether flags indicate a segment-head or a segment-tail
-        typename        FlagT,
-        typename        ReductionOp>
-    __device__ __forceinline__ T SegmentedReduce(
-        T               input,                  ///< [in] Calling thread's input
-        FlagT           flag,                   ///< [in] Whether or not the current lane is a segment head/tail
-        ReductionOp     reduction_op,           ///< [in] Reduction operator
-        Int2Type<true>  /*has_ballot*/)         ///< [in] Marker type for whether the target arch has ballot functionality
-    {
-        // Get the start flags for each thread in the warp.
-        int warp_flags = WARP_BALLOT(flag, member_mask);
-
-        if (!HEAD_SEGMENTED)
-            warp_flags <<= 1;
-
-        // Keep bits above the current thread.
-        warp_flags &= LaneMaskGt();
-
-        // Accommodate packing of multiple logical warps in a single physical warp
-        if (!IS_ARCH_WARP)
-        {
-            warp_flags >>= (LaneId() / LOGICAL_WARP_THREADS) * LOGICAL_WARP_THREADS;
-        }
-
-        // Find next flag
-        int next_flag = __clz(__brev(warp_flags));
-
-        // Clip the next segment at the warp boundary if necessary
-        if (LOGICAL_WARP_THREADS != 32)
-            next_flag = CUB_MIN(next_flag, LOGICAL_WARP_THREADS);
-
-        #pragma unroll
-        for (int STEP = 0; STEP < STEPS; STEP++)
-        {
-            const int OFFSET = 1 << STEP;
-
-            // Share input into buffer
-            ThreadStore<STORE_VOLATILE>(&temp_storage.reduce[lane_id], input);
-
-            WARP_SYNC(member_mask);
-
-            // Update input if peer_addend is in range
-            if (OFFSET + lane_id < next_flag)
-            {
-                T peer_addend = ThreadLoad<LOAD_VOLATILE>(&temp_storage.reduce[lane_id + OFFSET]);
-                input = reduction_op(input, peer_addend);
-            }
-
-            WARP_SYNC(member_mask);
-        }
-
-        return input;
-    }
-
-
-    /**
-     * Smem-based segmented reduce
-     */
-    template <
-        bool            HEAD_SEGMENTED,     ///< Whether flags indicate a segment-head or a segment-tail
-        typename        FlagT,
-        typename        ReductionOp>
-    __device__ __forceinline__ T SegmentedReduce(
-        T               input,                  ///< [in] Calling thread's input
-        FlagT           flag,                   ///< [in] Whether or not the current lane is a segment head/tail
-        ReductionOp     reduction_op,           ///< [in] Reduction operator
-        Int2Type<false> /*has_ballot*/)         ///< [in] Marker type for whether the target arch has ballot functionality
-    {
-        enum
-        {
-            UNSET   = 0x0,  // Is initially unset
-            SET     = 0x1,  // Is initially set
-            SEEN    = 0x2,  // Has seen another head flag from a successor peer
-        };
-
-        // Alias flags onto shared data storage
-        volatile SmemFlag *flag_storage = temp_storage.flags;
-
-        SmemFlag flag_status = (flag) ? SET : UNSET;
-
-        for (int STEP = 0; STEP < STEPS; STEP++)
-        {
-            const int OFFSET = 1 << STEP;
-
-            // Share input through buffer
-            ThreadStore<STORE_VOLATILE>(&temp_storage.reduce[lane_id], input);
-
-            WARP_SYNC(member_mask);
-
-            // Get peer from buffer
-            T peer_addend = ThreadLoad<LOAD_VOLATILE>(&temp_storage.reduce[lane_id + OFFSET]);
-
-            WARP_SYNC(member_mask);
-
-            // Share flag through buffer
-            flag_storage[lane_id] = flag_status;
-
-            // Get peer flag from buffer
-            SmemFlag peer_flag_status = flag_storage[lane_id + OFFSET];
-
-            // Update input if peer was in range
-            if (lane_id < LOGICAL_WARP_THREADS - OFFSET)
-            {
-                if (HEAD_SEGMENTED)
-                {
-                    // Head-segmented
-                    if ((flag_status & SEEN) == 0)
-                    {
-                        // Has not seen a more distant head flag
-                        if (peer_flag_status & SET)
-                        {
-                            // Has now seen a head flag
-                            flag_status |= SEEN;
-                        }
-                        else
-                        {
-                            // Peer is not a head flag: grab its count
-                            input = reduction_op(input, peer_addend);
-                        }
-
-                        // Update seen status to include that of peer
-                        flag_status |= (peer_flag_status & SEEN);
-                    }
-                }
-                else
-                {
-                    // Tail-segmented.  Simply propagate flag status
-                    if (!flag_status)
-                    {
-                        input = reduction_op(input, peer_addend);
-                        flag_status |= peer_flag_status;
-                    }
-
-                }
-            }
-        }
-
-        return input;
-    }
-
-
-    /******************************************************************************
-     * Interface
-     ******************************************************************************/
-
-    /**
-     * Reduction
-     */
-    template <
-        bool                ALL_LANES_VALID,        ///< Whether all lanes in each warp are contributing a valid fold of items
-        typename            ReductionOp>
-    __device__ __forceinline__ T Reduce(
-        T                   input,                  ///< [in] Calling thread's input
-        int                 valid_items,            ///< [in] Total number of valid items across the logical warp
-        ReductionOp         reduction_op)           ///< [in] Reduction operator
-    {
-        return ReduceStep<ALL_LANES_VALID>(input, valid_items, reduction_op, Int2Type<0>());
-    }
-
-
-    /**
-     * Segmented reduction
-     */
-    template <
-        bool            HEAD_SEGMENTED,     ///< Whether flags indicate a segment-head or a segment-tail
-        typename        FlagT,
-        typename        ReductionOp>
-    __device__ __forceinline__ T SegmentedReduce(
-        T               input,              ///< [in] Calling thread's input
-        FlagT            flag,               ///< [in] Whether or not the current lane is a segment head/tail
-        ReductionOp     reduction_op)       ///< [in] Reduction operator
-    {
-        return SegmentedReduce<HEAD_SEGMENTED>(input, flag, reduction_op, Int2Type<(PTX_ARCH >= 200)>());
-    }
-
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/warp/specializations/warp_scan_shfl.cuh b/pfsimulator/third_party/cub/warp/specializations/warp_scan_shfl.cuh
deleted file mode 100644
index 7f4e1c94b..000000000
--- a/pfsimulator/third_party/cub/warp/specializations/warp_scan_shfl.cuh
+++ /dev/null
@@ -1,632 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::WarpScanShfl provides SHFL-based variants of parallel prefix scan of items partitioned across a CUDA thread warp.
- */
-
-#pragma once
-
-#include "../../thread/thread_operators.cuh"
-#include "../../util_type.cuh"
-#include "../../util_ptx.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \brief WarpScanShfl provides SHFL-based variants of parallel prefix scan of items partitioned across a CUDA thread warp.
- *
- * LOGICAL_WARP_THREADS must be a power-of-two
- */
-template <
-    typename    T,                      ///< Data type being scanned
-    int         LOGICAL_WARP_THREADS,   ///< Number of threads per logical warp
-    int         PTX_ARCH>               ///< The PTX compute capability for which to to specialize this collective
-struct WarpScanShfl
-{
-    //---------------------------------------------------------------------
-    // Constants and type definitions
-    //---------------------------------------------------------------------
-
-    enum
-    {
-        /// Whether the logical warp size and the PTX warp size coincide
-        IS_ARCH_WARP = (LOGICAL_WARP_THREADS == CUB_WARP_THREADS(PTX_ARCH)),
-
-        /// The number of warp scan steps
-        STEPS = Log2<LOGICAL_WARP_THREADS>::VALUE,
-
-        /// The 5-bit SHFL mask for logically splitting warps into sub-segments starts 8-bits up
-        SHFL_C = (CUB_WARP_THREADS(PTX_ARCH) - LOGICAL_WARP_THREADS) << 8
-    };
-
-    template <typename S>
-    struct IntegerTraits
-    {
-        enum {
-            ///Whether the data type is a small (32b or less) integer for which we can use a single SFHL instruction per exchange
-            IS_SMALL_UNSIGNED = (Traits<S>::CATEGORY == UNSIGNED_INTEGER) && (sizeof(S) <= sizeof(unsigned int))
-        };
-    };
-
-    /// Shared memory storage layout type
-    struct TempStorage {};
-
-
-    //---------------------------------------------------------------------
-    // Thread fields
-    //---------------------------------------------------------------------
-
-    /// Lane index in logical warp
-    unsigned int lane_id;
-
-    /// Logical warp index in 32-thread physical warp
-    unsigned int warp_id;
-
-    /// 32-thread physical warp member mask of logical warp
-    unsigned int member_mask;
-
-    //---------------------------------------------------------------------
-    // Construction
-    //---------------------------------------------------------------------
-
-    /// Constructor
-    __device__ __forceinline__ WarpScanShfl(
-        TempStorage &/*temp_storage*/)
-    {
-        lane_id = LaneId();
-        warp_id = 0;
-        member_mask = 0xffffffffu >> (CUB_WARP_THREADS(PTX_ARCH) - LOGICAL_WARP_THREADS);
-
-        if (!IS_ARCH_WARP)
-        {
-            warp_id = lane_id / LOGICAL_WARP_THREADS;
-            lane_id = lane_id % LOGICAL_WARP_THREADS;
-            member_mask = member_mask << (warp_id * LOGICAL_WARP_THREADS);
-        }
-    }
-
-
-    //---------------------------------------------------------------------
-    // Inclusive scan steps
-    //---------------------------------------------------------------------
-
-    /// Inclusive prefix scan step (specialized for summation across int32 types)
-    __device__ __forceinline__ int InclusiveScanStep(
-        int             input,              ///< [in] Calling thread's input item.
-        cub::Sum        /*scan_op*/,        ///< [in] Binary scan operator
-        int             first_lane,         ///< [in] Index of first lane in segment
-        int             offset)             ///< [in] Up-offset to pull from
-    {
-        int output;
-        int shfl_c = first_lane | SHFL_C;   // Shuffle control (mask and first-lane)
-
-        // Use predicate set from SHFL to guard against invalid peers
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-        asm volatile(
-            "{"
-            "  .reg .s32 r0;"
-            "  .reg .pred p;"
-            "  shfl.sync.up.b32 r0|p, %1, %2, %3, %5;"
-            "  @p add.s32 r0, r0, %4;"
-            "  mov.s32 %0, r0;"
-            "}"
-            : "=r"(output) : "r"(input), "r"(offset), "r"(shfl_c), "r"(input), "r"(member_mask));
-#else
-        asm volatile(
-            "{"
-            "  .reg .s32 r0;"
-            "  .reg .pred p;"
-            "  shfl.up.b32 r0|p, %1, %2, %3;"
-            "  @p add.s32 r0, r0, %4;"
-            "  mov.s32 %0, r0;"
-            "}"
-            : "=r"(output) : "r"(input), "r"(offset), "r"(shfl_c), "r"(input));
-#endif
-
-        return output;
-    }
-
-    /// Inclusive prefix scan step (specialized for summation across uint32 types)
-    __device__ __forceinline__ unsigned int InclusiveScanStep(
-        unsigned int    input,              ///< [in] Calling thread's input item.
-        cub::Sum        /*scan_op*/,        ///< [in] Binary scan operator
-        int             first_lane,         ///< [in] Index of first lane in segment
-        int             offset)             ///< [in] Up-offset to pull from
-    {
-        unsigned int output;
-        int shfl_c = first_lane | SHFL_C;   // Shuffle control (mask and first-lane)
-
-        // Use predicate set from SHFL to guard against invalid peers
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-        asm volatile(
-            "{"
-            "  .reg .u32 r0;"
-            "  .reg .pred p;"
-            "  shfl.sync.up.b32 r0|p, %1, %2, %3, %5;"
-            "  @p add.u32 r0, r0, %4;"
-            "  mov.u32 %0, r0;"
-            "}"
-            : "=r"(output) : "r"(input), "r"(offset), "r"(shfl_c), "r"(input), "r"(member_mask));
-#else
-        asm volatile(
-            "{"
-            "  .reg .u32 r0;"
-            "  .reg .pred p;"
-            "  shfl.up.b32 r0|p, %1, %2, %3;"
-            "  @p add.u32 r0, r0, %4;"
-            "  mov.u32 %0, r0;"
-            "}"
-            : "=r"(output) : "r"(input), "r"(offset), "r"(shfl_c), "r"(input));
-#endif
-
-        return output;
-    }
-
-
-    /// Inclusive prefix scan step (specialized for summation across fp32 types)
-    __device__ __forceinline__ float InclusiveScanStep(
-        float           input,              ///< [in] Calling thread's input item.
-        cub::Sum        /*scan_op*/,        ///< [in] Binary scan operator
-        int             first_lane,         ///< [in] Index of first lane in segment
-        int             offset)             ///< [in] Up-offset to pull from
-    {
-        float output;
-        int shfl_c = first_lane | SHFL_C;   // Shuffle control (mask and first-lane)
-
-        // Use predicate set from SHFL to guard against invalid peers
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-        asm volatile(
-            "{"
-            "  .reg .f32 r0;"
-            "  .reg .pred p;"
-            "  shfl.sync.up.b32 r0|p, %1, %2, %3, %5;"
-            "  @p add.f32 r0, r0, %4;"
-            "  mov.f32 %0, r0;"
-            "}"
-            : "=f"(output) : "f"(input), "r"(offset), "r"(shfl_c), "f"(input), "r"(member_mask));
-#else
-        asm volatile(
-            "{"
-            "  .reg .f32 r0;"
-            "  .reg .pred p;"
-            "  shfl.up.b32 r0|p, %1, %2, %3;"
-            "  @p add.f32 r0, r0, %4;"
-            "  mov.f32 %0, r0;"
-            "}"
-            : "=f"(output) : "f"(input), "r"(offset), "r"(shfl_c), "f"(input));
-#endif
-
-        return output;
-    }
-
-
-    /// Inclusive prefix scan step (specialized for summation across unsigned long long types)
-    __device__ __forceinline__ unsigned long long InclusiveScanStep(
-        unsigned long long  input,              ///< [in] Calling thread's input item.
-        cub::Sum            /*scan_op*/,        ///< [in] Binary scan operator
-        int             first_lane,         ///< [in] Index of first lane in segment
-        int             offset)             ///< [in] Up-offset to pull from
-    {
-        unsigned long long output;
-        int shfl_c = first_lane | SHFL_C;   // Shuffle control (mask and first-lane)
-
-        // Use predicate set from SHFL to guard against invalid peers
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-        asm volatile(
-            "{"
-            "  .reg .u64 r0;"
-            "  .reg .u32 lo;"
-            "  .reg .u32 hi;"
-            "  .reg .pred p;"
-            "  mov.b64 {lo, hi}, %1;"
-            "  shfl.sync.up.b32 lo|p, lo, %2, %3, %5;"
-            "  shfl.sync.up.b32 hi|p, hi, %2, %3, %5;"
-            "  mov.b64 r0, {lo, hi};"
-            "  @p add.u64 r0, r0, %4;"
-            "  mov.u64 %0, r0;"
-            "}"
-            : "=l"(output) : "l"(input), "r"(offset), "r"(shfl_c), "l"(input), "r"(member_mask));
-#else
-        asm volatile(
-            "{"
-            "  .reg .u64 r0;"
-            "  .reg .u32 lo;"
-            "  .reg .u32 hi;"
-            "  .reg .pred p;"
-            "  mov.b64 {lo, hi}, %1;"
-            "  shfl.up.b32 lo|p, lo, %2, %3;"
-            "  shfl.up.b32 hi|p, hi, %2, %3;"
-            "  mov.b64 r0, {lo, hi};"
-            "  @p add.u64 r0, r0, %4;"
-            "  mov.u64 %0, r0;"
-            "}"
-            : "=l"(output) : "l"(input), "r"(offset), "r"(shfl_c), "l"(input));
-#endif
-
-        return output;
-    }
-
-
-    /// Inclusive prefix scan step (specialized for summation across long long types)
-    __device__ __forceinline__ long long InclusiveScanStep(
-        long long       input,              ///< [in] Calling thread's input item.
-        cub::Sum        /*scan_op*/,        ///< [in] Binary scan operator
-        int             first_lane,         ///< [in] Index of first lane in segment
-        int             offset)             ///< [in] Up-offset to pull from
-    {
-        long long output;
-        int shfl_c = first_lane | SHFL_C;   // Shuffle control (mask and first-lane)
-
-        // Use predicate set from SHFL to guard against invalid peers
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-        asm volatile(
-            "{"
-            "  .reg .s64 r0;"
-            "  .reg .u32 lo;"
-            "  .reg .u32 hi;"
-            "  .reg .pred p;"
-            "  mov.b64 {lo, hi}, %1;"
-            "  shfl.sync.up.b32 lo|p, lo, %2, %3, %5;"
-            "  shfl.sync.up.b32 hi|p, hi, %2, %3, %5;"
-            "  mov.b64 r0, {lo, hi};"
-            "  @p add.s64 r0, r0, %4;"
-            "  mov.s64 %0, r0;"
-            "}"
-            : "=l"(output) : "l"(input), "r"(offset), "r"(shfl_c), "l"(input), "r"(member_mask));
-#else
-        asm volatile(
-            "{"
-            "  .reg .s64 r0;"
-            "  .reg .u32 lo;"
-            "  .reg .u32 hi;"
-            "  .reg .pred p;"
-            "  mov.b64 {lo, hi}, %1;"
-            "  shfl.up.b32 lo|p, lo, %2, %3;"
-            "  shfl.up.b32 hi|p, hi, %2, %3;"
-            "  mov.b64 r0, {lo, hi};"
-            "  @p add.s64 r0, r0, %4;"
-            "  mov.s64 %0, r0;"
-            "}"
-            : "=l"(output) : "l"(input), "r"(offset), "r"(shfl_c), "l"(input));
-#endif
-
-        return output;
-    }
-
-
-    /// Inclusive prefix scan step (specialized for summation across fp64 types)
-    __device__ __forceinline__ double InclusiveScanStep(
-        double          input,              ///< [in] Calling thread's input item.
-        cub::Sum        /*scan_op*/,        ///< [in] Binary scan operator
-        int             first_lane,         ///< [in] Index of first lane in segment
-        int             offset)             ///< [in] Up-offset to pull from
-    {
-        double output;
-        int shfl_c = first_lane | SHFL_C;   // Shuffle control (mask and first-lane)
-
-        // Use predicate set from SHFL to guard against invalid peers
-#ifdef CUB_USE_COOPERATIVE_GROUPS
-        asm volatile(
-            "{"
-            "  .reg .u32 lo;"
-            "  .reg .u32 hi;"
-            "  .reg .pred p;"
-            "  .reg .f64 r0;"
-            "  mov.b64 %0, %1;"
-            "  mov.b64 {lo, hi}, %1;"
-            "  shfl.sync.up.b32 lo|p, lo, %2, %3, %4;"
-            "  shfl.sync.up.b32 hi|p, hi, %2, %3, %4;"
-            "  mov.b64 r0, {lo, hi};"
-            "  @p add.f64 %0, %0, r0;"
-            "}"
-            : "=d"(output) : "d"(input), "r"(offset), "r"(shfl_c), "r"(member_mask));
-#else
-        asm volatile(
-            "{"
-            "  .reg .u32 lo;"
-            "  .reg .u32 hi;"
-            "  .reg .pred p;"
-            "  .reg .f64 r0;"
-            "  mov.b64 %0, %1;"
-            "  mov.b64 {lo, hi}, %1;"
-            "  shfl.up.b32 lo|p, lo, %2, %3;"
-            "  shfl.up.b32 hi|p, hi, %2, %3;"
-            "  mov.b64 r0, {lo, hi};"
-            "  @p add.f64 %0, %0, r0;"
-            "}"
-            : "=d"(output) : "d"(input), "r"(offset), "r"(shfl_c));
-#endif
-
-        return output;
-    }
-
-
-/*
-    /// Inclusive prefix scan (specialized for ReduceBySegmentOp<cub::Sum> across KeyValuePair<OffsetT, Value> types)
-    template <typename Value, typename OffsetT>
-    __device__ __forceinline__ KeyValuePair<OffsetT, Value>InclusiveScanStep(
-        KeyValuePair<OffsetT, Value>    input,              ///< [in] Calling thread's input item.
-        ReduceBySegmentOp<cub::Sum>     scan_op,            ///< [in] Binary scan operator
-        int                             first_lane,         ///< [in] Index of first lane in segment
-        int                             offset)             ///< [in] Up-offset to pull from
-    {
-        KeyValuePair<OffsetT, Value> output;
-
-        output.value = InclusiveScanStep(input.value, cub::Sum(), first_lane, offset, Int2Type<IntegerTraits<Value>::IS_SMALL_UNSIGNED>());
-        output.key = InclusiveScanStep(input.key, cub::Sum(), first_lane, offset, Int2Type<IntegerTraits<OffsetT>::IS_SMALL_UNSIGNED>());
-
-        if (input.key > 0)
-            output.value = input.value;
-
-        return output;
-    }
-*/
-
-    /// Inclusive prefix scan step (generic)
-    template <typename _T, typename ScanOpT>
-    __device__ __forceinline__ _T InclusiveScanStep(
-        _T              input,              ///< [in] Calling thread's input item.
-        ScanOpT         scan_op,            ///< [in] Binary scan operator
-        int             first_lane,         ///< [in] Index of first lane in segment
-        int             offset)             ///< [in] Up-offset to pull from
-    {
-        _T temp = ShuffleUp<LOGICAL_WARP_THREADS>(input, offset, first_lane, member_mask);
-
-        // Perform scan op if from a valid peer
-        _T output = scan_op(temp, input);
-        if (static_cast<int>(lane_id) < first_lane + offset)
-            output = input;
-
-        return output;
-    }
-
-
-    /// Inclusive prefix scan step (specialized for small integers size 32b or less)
-    template <typename _T, typename ScanOpT>
-    __device__ __forceinline__ _T InclusiveScanStep(
-        _T              input,              ///< [in] Calling thread's input item.
-        ScanOpT         scan_op,            ///< [in] Binary scan operator
-        int             first_lane,         ///< [in] Index of first lane in segment
-        int             offset,             ///< [in] Up-offset to pull from
-        Int2Type<true>  /*is_small_unsigned*/)  ///< [in] Marker type indicating whether T is a small integer
-    {
-        return InclusiveScanStep(input, scan_op, first_lane, offset);
-    }
-
-
-    /// Inclusive prefix scan step (specialized for types other than small integers size 32b or less)
-    template <typename _T, typename ScanOpT>
-    __device__ __forceinline__ _T InclusiveScanStep(
-        _T              input,              ///< [in] Calling thread's input item.
-        ScanOpT          scan_op,            ///< [in] Binary scan operator
-        int             first_lane,         ///< [in] Index of first lane in segment
-        int             offset,             ///< [in] Up-offset to pull from
-        Int2Type<false> /*is_small_unsigned*/)  ///< [in] Marker type indicating whether T is a small integer
-    {
-        return InclusiveScanStep(input, scan_op, first_lane, offset);
-    }
-
-
-    /******************************************************************************
-     * Interface
-     ******************************************************************************/
-
-    //---------------------------------------------------------------------
-    // Broadcast
-    //---------------------------------------------------------------------
-
-    /// Broadcast
-    __device__ __forceinline__ T Broadcast(
-        T               input,              ///< [in] The value to broadcast
-        int             src_lane)           ///< [in] Which warp lane is to do the broadcasting
-    {
-        return ShuffleIndex<LOGICAL_WARP_THREADS>(input, src_lane, member_mask);
-    }
-
-
-    //---------------------------------------------------------------------
-    // Inclusive operations
-    //---------------------------------------------------------------------
-
-    /// Inclusive scan
-    template <typename _T, typename ScanOpT>
-    __device__ __forceinline__ void InclusiveScan(
-        _T              input,              ///< [in] Calling thread's input item.
-        _T              &inclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
-        ScanOpT         scan_op)            ///< [in] Binary scan operator
-    {
-        inclusive_output = input;
-
-        // Iterate scan steps
-        int segment_first_lane = 0;
-
-        // Iterate scan steps
-        #pragma unroll
-        for (int STEP = 0; STEP < STEPS; STEP++)
-        {
-            inclusive_output = InclusiveScanStep(
-                inclusive_output,
-                scan_op,
-                segment_first_lane,
-                (1 << STEP),
-                Int2Type<IntegerTraits<T>::IS_SMALL_UNSIGNED>());
-        }
-
-    }
-
-    /// Inclusive scan, specialized for reduce-value-by-key
-    template <typename KeyT, typename ValueT, typename ReductionOpT>
-    __device__ __forceinline__ void InclusiveScan(
-        KeyValuePair<KeyT, ValueT>      input,              ///< [in] Calling thread's input item.
-        KeyValuePair<KeyT, ValueT>      &inclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
-        ReduceByKeyOp<ReductionOpT >    scan_op)            ///< [in] Binary scan operator
-    {
-        inclusive_output = input;
-
-        KeyT pred_key = ShuffleUp<LOGICAL_WARP_THREADS>(inclusive_output.key, 1, 0, member_mask);
-
-        unsigned int ballot = WARP_BALLOT((pred_key != inclusive_output.key), member_mask);
-
-        // Mask away all lanes greater than ours
-        ballot = ballot & LaneMaskLe();
-
-        // Find index of first set bit
-        int segment_first_lane = CUB_MAX(0, 31 - __clz(ballot));
-
-        // Iterate scan steps
-        #pragma unroll
-        for (int STEP = 0; STEP < STEPS; STEP++)
-        {
-            inclusive_output.value = InclusiveScanStep(
-                inclusive_output.value,
-                scan_op.op,
-                segment_first_lane,
-                (1 << STEP),
-                Int2Type<IntegerTraits<T>::IS_SMALL_UNSIGNED>());
-        }
-    }
-
-
-    /// Inclusive scan with aggregate
-    template <typename ScanOpT>
-    __device__ __forceinline__ void InclusiveScan(
-        T               input,              ///< [in] Calling thread's input item.
-        T               &inclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
-        ScanOpT         scan_op,            ///< [in] Binary scan operator
-        T               &warp_aggregate)    ///< [out] Warp-wide aggregate reduction of input items.
-    {
-        InclusiveScan(input, inclusive_output, scan_op);
-
-        // Grab aggregate from last warp lane
-        warp_aggregate = ShuffleIndex<LOGICAL_WARP_THREADS>(inclusive_output, LOGICAL_WARP_THREADS - 1, member_mask);
-    }
-
-
-    //---------------------------------------------------------------------
-    // Get exclusive from inclusive
-    //---------------------------------------------------------------------
-
-    /// Update inclusive and exclusive using input and inclusive
-    template <typename ScanOpT, typename IsIntegerT>
-    __device__ __forceinline__ void Update(
-        T                       /*input*/,          ///< [in]
-        T                       &inclusive,         ///< [in, out]
-        T                       &exclusive,         ///< [out]
-        ScanOpT                 /*scan_op*/,        ///< [in]
-        IsIntegerT              /*is_integer*/)     ///< [in]
-    {
-        // initial value unknown
-        exclusive = ShuffleUp<LOGICAL_WARP_THREADS>(inclusive, 1, 0, member_mask);
-    }
-
-    /// Update inclusive and exclusive using input and inclusive (specialized for summation of integer types)
-    __device__ __forceinline__ void Update(
-        T                       input,
-        T                       &inclusive,
-        T                       &exclusive,
-        cub::Sum                /*scan_op*/,
-        Int2Type<true>          /*is_integer*/)
-    {
-        // initial value presumed 0
-        exclusive = inclusive - input;
-    }
-
-    /// Update inclusive and exclusive using initial value using input, inclusive, and initial value
-    template <typename ScanOpT, typename IsIntegerT>
-    __device__ __forceinline__ void Update (
-        T                       /*input*/,
-        T                       &inclusive,
-        T                       &exclusive,
-        ScanOpT                 scan_op,
-        T                       initial_value,
-        IsIntegerT              /*is_integer*/)
-    {
-        inclusive = scan_op(initial_value, inclusive);
-        exclusive = ShuffleUp<LOGICAL_WARP_THREADS>(inclusive, 1, 0, member_mask);
-
-        if (lane_id == 0)
-            exclusive = initial_value;
-    }
-
-    /// Update inclusive and exclusive using initial value using input and inclusive (specialized for summation of integer types)
-    __device__ __forceinline__ void Update (
-        T                       input,
-        T                       &inclusive,
-        T                       &exclusive,
-        cub::Sum                scan_op,
-        T                       initial_value,
-        Int2Type<true>          /*is_integer*/)
-    {
-        inclusive = scan_op(initial_value, inclusive);
-        exclusive = inclusive - input;
-    }
-
-
-    /// Update inclusive, exclusive, and warp aggregate using input and inclusive
-    template <typename ScanOpT, typename IsIntegerT>
-    __device__ __forceinline__ void Update (
-        T                       input,
-        T                       &inclusive,
-        T                       &exclusive,
-        T                       &warp_aggregate,
-        ScanOpT                 scan_op,
-        IsIntegerT              is_integer)
-    {
-        warp_aggregate = ShuffleIndex<LOGICAL_WARP_THREADS>(inclusive, LOGICAL_WARP_THREADS - 1, member_mask);
-        Update(input, inclusive, exclusive, scan_op, is_integer);
-    }
-
-    /// Update inclusive, exclusive, and warp aggregate using input, inclusive, and initial value
-    template <typename ScanOpT, typename IsIntegerT>
-    __device__ __forceinline__ void Update (
-        T                       input,
-        T                       &inclusive,
-        T                       &exclusive,
-        T                       &warp_aggregate,
-        ScanOpT                 scan_op,
-        T                       initial_value,
-        IsIntegerT              is_integer)
-    {
-        warp_aggregate = ShuffleIndex<LOGICAL_WARP_THREADS>(inclusive, LOGICAL_WARP_THREADS - 1, member_mask);
-        Update(input, inclusive, exclusive, scan_op, initial_value, is_integer);
-    }
-
-
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/warp/specializations/warp_scan_smem.cuh b/pfsimulator/third_party/cub/warp/specializations/warp_scan_smem.cuh
deleted file mode 100644
index 3237fcbfe..000000000
--- a/pfsimulator/third_party/cub/warp/specializations/warp_scan_smem.cuh
+++ /dev/null
@@ -1,397 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * cub::WarpScanSmem provides smem-based variants of parallel prefix scan of items partitioned across a CUDA thread warp.
- */
-
-#pragma once
-
-#include "../../thread/thread_operators.cuh"
-#include "../../thread/thread_load.cuh"
-#include "../../thread/thread_store.cuh"
-#include "../../util_type.cuh"
-#include "../../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \brief WarpScanSmem provides smem-based variants of parallel prefix scan of items partitioned across a CUDA thread warp.
- */
-template <
-    typename    T,                      ///< Data type being scanned
-    int         LOGICAL_WARP_THREADS,   ///< Number of threads per logical warp
-    int         PTX_ARCH>               ///< The PTX compute capability for which to to specialize this collective
-struct WarpScanSmem
-{
-    /******************************************************************************
-     * Constants and type definitions
-     ******************************************************************************/
-
-    enum
-    {
-        /// Whether the logical warp size and the PTX warp size coincide
-        IS_ARCH_WARP = (LOGICAL_WARP_THREADS == CUB_WARP_THREADS(PTX_ARCH)),
-
-        /// Whether the logical warp size is a power-of-two
-        IS_POW_OF_TWO = PowerOfTwo<LOGICAL_WARP_THREADS>::VALUE,
-
-        /// The number of warp scan steps
-        STEPS = Log2<LOGICAL_WARP_THREADS>::VALUE,
-
-        /// The number of threads in half a warp
-        HALF_WARP_THREADS = 1 << (STEPS - 1),
-
-        /// The number of shared memory elements per warp
-        WARP_SMEM_ELEMENTS =  LOGICAL_WARP_THREADS + HALF_WARP_THREADS,
-    };
-
-    /// Storage cell type (workaround for SM1x compiler bugs with custom-ops like Max() on signed chars)
-    typedef typename If<((Equals<T, char>::VALUE || Equals<T, signed char>::VALUE) && (PTX_ARCH < 200)), int, T>::Type CellT;
-
-    /// Shared memory storage layout type (1.5 warps-worth of elements for each warp)
-    typedef CellT _TempStorage[WARP_SMEM_ELEMENTS];
-
-    // Alias wrapper allowing storage to be unioned
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    /******************************************************************************
-     * Thread fields
-     ******************************************************************************/
-
-    _TempStorage    &temp_storage;
-    unsigned int    lane_id;
-    unsigned int    member_mask;
-
-
-    /******************************************************************************
-     * Construction
-     ******************************************************************************/
-
-    /// Constructor
-    __device__ __forceinline__ WarpScanSmem(
-        TempStorage     &temp_storage)
-    :
-        temp_storage(temp_storage.Alias()),
-
-        lane_id(IS_ARCH_WARP ?
-            LaneId() :
-            LaneId() % LOGICAL_WARP_THREADS),
-
-        member_mask((0xffffffff >> (32 - LOGICAL_WARP_THREADS)) << ((IS_ARCH_WARP || !IS_POW_OF_TWO ) ?
-            0 : // arch-width and non-power-of-two subwarps cannot be tiled with the arch-warp
-            ((LaneId() / LOGICAL_WARP_THREADS) * LOGICAL_WARP_THREADS)))
-    {}
-
-
-    /******************************************************************************
-     * Utility methods
-     ******************************************************************************/
-
-    /// Basic inclusive scan iteration (template unrolled, inductive-case specialization)
-    template <
-        bool        HAS_IDENTITY,
-        int         STEP,
-        typename    ScanOp>
-    __device__ __forceinline__ void ScanStep(
-        T                       &partial,
-        ScanOp                  scan_op,
-        Int2Type<STEP>          /*step*/)
-    {
-        const int OFFSET = 1 << STEP;
-
-        // Share partial into buffer
-        ThreadStore<STORE_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id], (CellT) partial);
-
-        WARP_SYNC(member_mask);
-
-        // Update partial if addend is in range
-        if (HAS_IDENTITY || (lane_id >= OFFSET))
-        {
-            T addend = (T) ThreadLoad<LOAD_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id - OFFSET]);
-            partial = scan_op(addend, partial);
-        }
-        WARP_SYNC(member_mask);
-
-        ScanStep<HAS_IDENTITY>(partial, scan_op, Int2Type<STEP + 1>());
-    }
-
-
-    /// Basic inclusive scan iteration(template unrolled, base-case specialization)
-    template <
-        bool        HAS_IDENTITY,
-        typename    ScanOp>
-    __device__ __forceinline__ void ScanStep(
-        T                       &/*partial*/,
-        ScanOp                  /*scan_op*/,
-        Int2Type<STEPS>         /*step*/)
-    {}
-
-
-    /// Inclusive prefix scan (specialized for summation across primitive types)
-    __device__ __forceinline__ void InclusiveScan(
-        T                       input,              ///< [in] Calling thread's input item.
-        T                       &output,            ///< [out] Calling thread's output item.  May be aliased with \p input.
-        Sum                     scan_op,            ///< [in] Binary scan operator
-        Int2Type<true>          /*is_primitive*/)   ///< [in] Marker type indicating whether T is primitive type
-    {
-        T identity = 0;
-        ThreadStore<STORE_VOLATILE>(&temp_storage[lane_id], (CellT) identity);
-
-        WARP_SYNC(member_mask);
-
-        // Iterate scan steps
-        output = input;
-        ScanStep<true>(output, scan_op, Int2Type<0>());
-    }
-
-
-    /// Inclusive prefix scan
-    template <typename ScanOp, int IS_PRIMITIVE>
-    __device__ __forceinline__ void InclusiveScan(
-        T                       input,              ///< [in] Calling thread's input item.
-        T                       &output,            ///< [out] Calling thread's output item.  May be aliased with \p input.
-        ScanOp                  scan_op,            ///< [in] Binary scan operator
-        Int2Type<IS_PRIMITIVE>  /*is_primitive*/)   ///< [in] Marker type indicating whether T is primitive type
-    {
-        // Iterate scan steps
-        output = input;
-        ScanStep<false>(output, scan_op, Int2Type<0>());
-    }
-
-
-    /******************************************************************************
-     * Interface
-     ******************************************************************************/
-
-    //---------------------------------------------------------------------
-    // Broadcast
-    //---------------------------------------------------------------------
-
-    /// Broadcast
-    __device__ __forceinline__ T Broadcast(
-        T               input,              ///< [in] The value to broadcast
-        unsigned int    src_lane)           ///< [in] Which warp lane is to do the broadcasting
-    {
-        if (lane_id == src_lane)
-        {
-            ThreadStore<STORE_VOLATILE>(temp_storage, (CellT) input);
-        }
-
-        WARP_SYNC(member_mask);
-
-        return (T)ThreadLoad<LOAD_VOLATILE>(temp_storage);
-    }
-
-
-    //---------------------------------------------------------------------
-    // Inclusive operations
-    //---------------------------------------------------------------------
-
-    /// Inclusive scan
-    template <typename ScanOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T               input,              ///< [in] Calling thread's input item.
-        T               &inclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
-        ScanOp          scan_op)            ///< [in] Binary scan operator
-    {
-        InclusiveScan(input, inclusive_output, scan_op, Int2Type<Traits<T>::PRIMITIVE>());
-    }
-
-
-    /// Inclusive scan with aggregate
-    template <typename ScanOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T               input,              ///< [in] Calling thread's input item.
-        T               &inclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
-        ScanOp          scan_op,            ///< [in] Binary scan operator
-        T               &warp_aggregate)    ///< [out] Warp-wide aggregate reduction of input items.
-    {
-        InclusiveScan(input, inclusive_output, scan_op);
-
-        // Retrieve aggregate
-        ThreadStore<STORE_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id], (CellT) inclusive_output);
-
-        WARP_SYNC(member_mask);
-
-        warp_aggregate = (T) ThreadLoad<LOAD_VOLATILE>(&temp_storage[WARP_SMEM_ELEMENTS - 1]);
-
-        WARP_SYNC(member_mask);
-    }
-
-
-    //---------------------------------------------------------------------
-    // Get exclusive from inclusive
-    //---------------------------------------------------------------------
-
-    /// Update inclusive and exclusive using input and inclusive
-    template <typename ScanOpT, typename IsIntegerT>
-    __device__ __forceinline__ void Update(
-        T                       /*input*/,      ///< [in]
-        T                       &inclusive,     ///< [in, out]
-        T                       &exclusive,     ///< [out]
-        ScanOpT                 /*scan_op*/,    ///< [in]
-        IsIntegerT              /*is_integer*/) ///< [in]
-    {
-        // initial value unknown
-        ThreadStore<STORE_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id], (CellT) inclusive);
-
-        WARP_SYNC(member_mask);
-
-        exclusive = (T) ThreadLoad<LOAD_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id - 1]);
-    }
-
-    /// Update inclusive and exclusive using input and inclusive (specialized for summation of integer types)
-    __device__ __forceinline__ void Update(
-        T                       input,
-        T                       &inclusive,
-        T                       &exclusive,
-        cub::Sum                /*scan_op*/,
-        Int2Type<true>          /*is_integer*/)
-    {
-        // initial value presumed 0
-        exclusive = inclusive - input;
-    }
-
-    /// Update inclusive and exclusive using initial value using input, inclusive, and initial value
-    template <typename ScanOpT, typename IsIntegerT>
-    __device__ __forceinline__ void Update (
-        T                       /*input*/,
-        T                       &inclusive,
-        T                       &exclusive,
-        ScanOpT                 scan_op,
-        T                       initial_value,
-        IsIntegerT              /*is_integer*/)
-    {
-        inclusive = scan_op(initial_value, inclusive);
-        ThreadStore<STORE_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id], (CellT) inclusive);
-
-        WARP_SYNC(member_mask);
-
-        exclusive = (T) ThreadLoad<LOAD_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id - 1]);
-        if (lane_id == 0)
-            exclusive = initial_value;
-    }
-
-    /// Update inclusive and exclusive using initial value using input and inclusive (specialized for summation of integer types)
-    __device__ __forceinline__ void Update (
-        T                       input,
-        T                       &inclusive,
-        T                       &exclusive,
-        cub::Sum                scan_op,
-        T                       initial_value,
-        Int2Type<true>          /*is_integer*/)
-    {
-        inclusive = scan_op(initial_value, inclusive);
-        exclusive = inclusive - input;
-    }
-
-
-    /// Update inclusive, exclusive, and warp aggregate using input and inclusive
-    template <typename ScanOpT, typename IsIntegerT>
-    __device__ __forceinline__ void Update (
-        T                       /*input*/,
-        T                       &inclusive,
-        T                       &exclusive,
-        T                       &warp_aggregate,
-        ScanOpT                 /*scan_op*/,
-        IsIntegerT              /*is_integer*/)
-    {
-        // Initial value presumed to be unknown or identity (either way our padding is correct)
-        ThreadStore<STORE_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id], (CellT) inclusive);
-
-        WARP_SYNC(member_mask);
-
-        exclusive = (T) ThreadLoad<LOAD_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id - 1]);
-        warp_aggregate = (T) ThreadLoad<LOAD_VOLATILE>(&temp_storage[WARP_SMEM_ELEMENTS - 1]);
-    }
-
-    /// Update inclusive, exclusive, and warp aggregate using input and inclusive (specialized for summation of integer types)
-    __device__ __forceinline__ void Update (
-        T                       input,
-        T                       &inclusive,
-        T                       &exclusive,
-        T                       &warp_aggregate,
-        cub::Sum                /*scan_o*/,
-        Int2Type<true>          /*is_integer*/)
-    {
-        // Initial value presumed to be unknown or identity (either way our padding is correct)
-        ThreadStore<STORE_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id], (CellT) inclusive);
-
-        WARP_SYNC(member_mask);
-
-        warp_aggregate = (T) ThreadLoad<LOAD_VOLATILE>(&temp_storage[WARP_SMEM_ELEMENTS - 1]);
-        exclusive = inclusive - input;
-    }
-
-    /// Update inclusive, exclusive, and warp aggregate using input, inclusive, and initial value
-    template <typename ScanOpT, typename IsIntegerT>
-    __device__ __forceinline__ void Update (
-        T                       /*input*/,
-        T                       &inclusive,
-        T                       &exclusive,
-        T                       &warp_aggregate,
-        ScanOpT                 scan_op,
-        T                       initial_value,
-        IsIntegerT              /*is_integer*/)
-    {
-        // Broadcast warp aggregate
-        ThreadStore<STORE_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id], (CellT) inclusive);
-
-        WARP_SYNC(member_mask);
-
-        warp_aggregate = (T) ThreadLoad<LOAD_VOLATILE>(&temp_storage[WARP_SMEM_ELEMENTS - 1]);
-
-        WARP_SYNC(member_mask);
-
-        // Update inclusive with initial value
-        inclusive = scan_op(initial_value, inclusive);
-
-        // Get exclusive from exclusive
-        ThreadStore<STORE_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id - 1], (CellT) inclusive);
-
-        WARP_SYNC(member_mask);
-
-        exclusive = (T) ThreadLoad<LOAD_VOLATILE>(&temp_storage[HALF_WARP_THREADS + lane_id - 2]);
-
-        if (lane_id == 0)
-            exclusive = initial_value;
-    }
-
-
-};
-
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/warp/warp_reduce.cuh b/pfsimulator/third_party/cub/warp/warp_reduce.cuh
deleted file mode 100644
index 189896b07..000000000
--- a/pfsimulator/third_party/cub/warp/warp_reduce.cuh
+++ /dev/null
@@ -1,612 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * The cub::WarpReduce class provides [<em>collective</em>](index.html#sec0) methods for computing a parallel reduction of items partitioned across a CUDA thread warp.
- */
-
-#pragma once
-
-#include "specializations/warp_reduce_shfl.cuh"
-#include "specializations/warp_reduce_smem.cuh"
-#include "../thread/thread_operators.cuh"
-#include "../util_arch.cuh"
-#include "../util_type.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-
-/**
- * \addtogroup WarpModule
- * @{
- */
-
-/**
- * \brief The WarpReduce class provides [<em>collective</em>](index.html#sec0) methods for computing a parallel reduction of items partitioned across a CUDA thread warp. ![](warp_reduce_logo.png)
- *
- * \tparam T                        The reduction input/output element type
- * \tparam LOGICAL_WARP_THREADS     <b>[optional]</b> The number of threads per "logical" warp (may be less than the number of hardware warp threads).  Default is the warp size of the targeted CUDA compute-capability (e.g., 32 threads for SM20).
- * \tparam PTX_ARCH                 <b>[optional]</b> \ptxversion
- *
- * \par Overview
- * - A <a href="http://en.wikipedia.org/wiki/Reduce_(higher-order_function)"><em>reduction</em></a> (or <em>fold</em>)
- *   uses a binary combining operator to compute a single aggregate from a list of input elements.
- * - Supports "logical" warps smaller than the physical warp size (e.g., logical warps of 8 threads)
- * - The number of entrant threads must be an multiple of \p LOGICAL_WARP_THREADS
- *
- * \par Performance Considerations
- * - Uses special instructions when applicable (e.g., warp \p SHFL instructions)
- * - Uses synchronization-free communication between warp lanes when applicable
- * - Incurs zero bank conflicts for most types
- * - Computation is slightly more efficient (i.e., having lower instruction overhead) for:
- *     - Summation (<b><em>vs.</em></b> generic reduction)
- *     - The architecture's warp size is a whole multiple of \p LOGICAL_WARP_THREADS
- *
- * \par Simple Examples
- * \warpcollective{WarpReduce}
- * \par
- * The code snippet below illustrates four concurrent warp sum reductions within a block of
- * 128 threads (one per each of the 32-thread warps).
- * \par
- * \code
- * #include <cub/cub.cuh>
- *
- * __global__ void ExampleKernel(...)
- * {
- *     // Specialize WarpReduce for type int
- *     typedef cub::WarpReduce<int> WarpReduce;
- *
- *     // Allocate WarpReduce shared memory for 4 warps
- *     __shared__ typename WarpReduce::TempStorage temp_storage[4];
- *
- *     // Obtain one input item per thread
- *     int thread_data = ...
- *
- *     // Return the warp-wide sums to each lane0 (threads 0, 32, 64, and 96)
- *     int warp_id = threadIdx.x / 32;
- *     int aggregate = WarpReduce(temp_storage[warp_id]).Sum(thread_data);
- *
- * \endcode
- * \par
- * Suppose the set of input \p thread_data across the block of threads is <tt>{0, 1, 2, 3, ..., 127}</tt>.
- * The corresponding output \p aggregate in threads 0, 32, 64, and 96 will \p 496, \p 1520,
- * \p 2544, and \p 3568, respectively (and is undefined in other threads).
- *
- * \par
- * The code snippet below illustrates a single warp sum reduction within a block of
- * 128 threads.
- * \par
- * \code
- * #include <cub/cub.cuh>
- *
- * __global__ void ExampleKernel(...)
- * {
- *     // Specialize WarpReduce for type int
- *     typedef cub::WarpReduce<int> WarpReduce;
- *
- *     // Allocate WarpReduce shared memory for one warp
- *     __shared__ typename WarpReduce::TempStorage temp_storage;
- *     ...
- *
- *     // Only the first warp performs a reduction
- *     if (threadIdx.x < 32)
- *     {
- *         // Obtain one input item per thread
- *         int thread_data = ...
- *
- *         // Return the warp-wide sum to lane0
- *         int aggregate = WarpReduce(temp_storage).Sum(thread_data);
- *
- * \endcode
- * \par
- * Suppose the set of input \p thread_data across the warp of threads is <tt>{0, 1, 2, 3, ..., 31}</tt>.
- * The corresponding output \p aggregate in thread0 will be \p 496 (and is undefined in other threads).
- *
- */
-template <
-    typename    T,
-    int         LOGICAL_WARP_THREADS    = CUB_PTX_WARP_THREADS,
-    int         PTX_ARCH                = CUB_PTX_ARCH>
-class WarpReduce
-{
-private:
-
-    /******************************************************************************
-     * Constants and type definitions
-     ******************************************************************************/
-
-    enum
-    {
-        /// Whether the logical warp size and the PTX warp size coincide
-        IS_ARCH_WARP = (LOGICAL_WARP_THREADS == CUB_WARP_THREADS(PTX_ARCH)),
-
-        /// Whether the logical warp size is a power-of-two
-        IS_POW_OF_TWO = PowerOfTwo<LOGICAL_WARP_THREADS>::VALUE,
-    };
-
-public:
-
-    #ifndef DOXYGEN_SHOULD_SKIP_THIS    // Do not document
-
-    /// Internal specialization.  Use SHFL-based reduction if (architecture is >= SM30) and (LOGICAL_WARP_THREADS is a power-of-two)
-    typedef typename If<(PTX_ARCH >= 300) && (IS_POW_OF_TWO),
-        WarpReduceShfl<T, LOGICAL_WARP_THREADS, PTX_ARCH>,
-        WarpReduceSmem<T, LOGICAL_WARP_THREADS, PTX_ARCH> >::Type InternalWarpReduce;
-
-    #endif // DOXYGEN_SHOULD_SKIP_THIS
-
-
-private:
-
-    /// Shared memory storage layout type for WarpReduce
-    typedef typename InternalWarpReduce::TempStorage _TempStorage;
-
-
-    /******************************************************************************
-     * Thread fields
-     ******************************************************************************/
-
-    /// Shared storage reference
-    _TempStorage &temp_storage;
-
-
-    /******************************************************************************
-     * Utility methods
-     ******************************************************************************/
-
-public:
-
-    /// \smemstorage{WarpReduce}
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    /******************************************************************//**
-     * \name Collective constructors
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Collective constructor using the specified memory allocation as temporary storage.  Logical warp and lane identifiers are constructed from <tt>threadIdx.x</tt>.
-     */
-    __device__ __forceinline__ WarpReduce(
-        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
-    :
-        temp_storage(temp_storage.Alias())
-    {}
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Summation reductions
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Computes a warp-wide sum in the calling warp.  The output is valid in warp <em>lane</em><sub>0</sub>.
-     *
-     * \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates four concurrent warp sum reductions within a block of
-     * 128 threads (one per each of the 32-thread warps).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpReduce for type int
-     *     typedef cub::WarpReduce<int> WarpReduce;
-     *
-     *     // Allocate WarpReduce shared memory for 4 warps
-     *     __shared__ typename WarpReduce::TempStorage temp_storage[4];
-     *
-     *     // Obtain one input item per thread
-     *     int thread_data = ...
-     *
-     *     // Return the warp-wide sums to each lane0
-     *     int warp_id = threadIdx.x / 32;
-     *     int aggregate = WarpReduce(temp_storage[warp_id]).Sum(thread_data);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, 1, 2, 3, ..., 127}</tt>.
-     * The corresponding output \p aggregate in threads 0, 32, 64, and 96 will \p 496, \p 1520,
-     * \p 2544, and \p 3568, respectively (and is undefined in other threads).
-     *
-     */
-    __device__ __forceinline__ T Sum(
-        T                   input)              ///< [in] Calling thread's input
-    {
-        return InternalWarpReduce(temp_storage).template Reduce<true>(input, LOGICAL_WARP_THREADS, cub::Sum());
-    }
-
-    /**
-     * \brief Computes a partially-full warp-wide sum in the calling warp.  The output is valid in warp <em>lane</em><sub>0</sub>.
-     *
-     * All threads across the calling warp must agree on the same value for \p valid_items.  Otherwise the result is undefined.
-     *
-     * \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a sum reduction within a single, partially-full
-     * block of 32 threads (one warp).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(int *d_data, int valid_items)
-     * {
-     *     // Specialize WarpReduce for type int
-     *     typedef cub::WarpReduce<int> WarpReduce;
-     *
-     *     // Allocate WarpReduce shared memory for one warp
-     *     __shared__ typename WarpReduce::TempStorage temp_storage;
-     *
-     *     // Obtain one input item per thread if in range
-     *     int thread_data;
-     *     if (threadIdx.x < valid_items)
-     *         thread_data = d_data[threadIdx.x];
-     *
-     *     // Return the warp-wide sums to each lane0
-     *     int aggregate = WarpReduce(temp_storage).Sum(
-     *         thread_data, valid_items);
-     *
-     * \endcode
-     * \par
-     * Suppose the input \p d_data is <tt>{0, 1, 2, 3, 4, ...</tt> and \p valid_items
-     * is \p 4.  The corresponding output \p aggregate in thread0 is \p 6 (and is
-     * undefined in other threads).
-     *
-     */
-    __device__ __forceinline__ T Sum(
-        T                   input,              ///< [in] Calling thread's input
-        int                 valid_items)        ///< [in] Total number of valid items in the calling thread's logical warp (may be less than \p LOGICAL_WARP_THREADS)
-    {
-        // Determine if we don't need bounds checking
-        return InternalWarpReduce(temp_storage).template Reduce<false>(input, valid_items, cub::Sum());
-    }
-
-
-    /**
-     * \brief Computes a segmented sum in the calling warp where segments are defined by head-flags.  The sum of each segment is returned to the first lane in that segment (which always includes <em>lane</em><sub>0</sub>).
-     *
-     * \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a head-segmented warp sum
-     * reduction within a block of 32 threads (one warp).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpReduce for type int
-     *     typedef cub::WarpReduce<int> WarpReduce;
-     *
-     *     // Allocate WarpReduce shared memory for one warp
-     *     __shared__ typename WarpReduce::TempStorage temp_storage;
-     *
-     *     // Obtain one input item and flag per thread
-     *     int thread_data = ...
-     *     int head_flag = ...
-     *
-     *     // Return the warp-wide sums to each lane0
-     *     int aggregate = WarpReduce(temp_storage).HeadSegmentedSum(
-     *         thread_data, head_flag);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data and \p head_flag across the block of threads
-     * is <tt>{0, 1, 2, 3, ..., 31</tt> and is <tt>{1, 0, 0, 0, 1, 0, 0, 0, ..., 1, 0, 0, 0</tt>,
-     * respectively.  The corresponding output \p aggregate in threads 0, 4, 8, etc. will be
-     * \p 6, \p 22, \p 38, etc. (and is undefined in other threads).
-     *
-     * \tparam ReductionOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
-     *
-     */
-    template <
-        typename            FlagT>
-    __device__ __forceinline__ T HeadSegmentedSum(
-        T                   input,              ///< [in] Calling thread's input
-        FlagT                head_flag)          ///< [in] Head flag denoting whether or not \p input is the start of a new segment
-    {
-        return HeadSegmentedReduce(input, head_flag, cub::Sum());
-    }
-
-
-    /**
-     * \brief Computes a segmented sum in the calling warp where segments are defined by tail-flags.  The sum of each segment is returned to the first lane in that segment (which always includes <em>lane</em><sub>0</sub>).
-     *
-     * \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a tail-segmented warp sum
-     * reduction within a block of 32 threads (one warp).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpReduce for type int
-     *     typedef cub::WarpReduce<int> WarpReduce;
-     *
-     *     // Allocate WarpReduce shared memory for one warp
-     *     __shared__ typename WarpReduce::TempStorage temp_storage;
-     *
-     *     // Obtain one input item and flag per thread
-     *     int thread_data = ...
-     *     int tail_flag = ...
-     *
-     *     // Return the warp-wide sums to each lane0
-     *     int aggregate = WarpReduce(temp_storage).TailSegmentedSum(
-     *         thread_data, tail_flag);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data and \p tail_flag across the block of threads
-     * is <tt>{0, 1, 2, 3, ..., 31</tt> and is <tt>{0, 0, 0, 1, 0, 0, 0, 1, ..., 0, 0, 0, 1</tt>,
-     * respectively.  The corresponding output \p aggregate in threads 0, 4, 8, etc. will be
-     * \p 6, \p 22, \p 38, etc. (and is undefined in other threads).
-     *
-     * \tparam ReductionOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <
-        typename            FlagT>
-    __device__ __forceinline__ T TailSegmentedSum(
-        T                   input,              ///< [in] Calling thread's input
-        FlagT                tail_flag)          ///< [in] Head flag denoting whether or not \p input is the start of a new segment
-    {
-        return TailSegmentedReduce(input, tail_flag, cub::Sum());
-    }
-
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Generic reductions
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Computes a warp-wide reduction in the calling warp using the specified binary reduction functor.  The output is valid in warp <em>lane</em><sub>0</sub>.
-     *
-     * Supports non-commutative reduction operators
-     *
-     * \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates four concurrent warp max reductions within a block of
-     * 128 threads (one per each of the 32-thread warps).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpReduce for type int
-     *     typedef cub::WarpReduce<int> WarpReduce;
-     *
-     *     // Allocate WarpReduce shared memory for 4 warps
-     *     __shared__ typename WarpReduce::TempStorage temp_storage[4];
-     *
-     *     // Obtain one input item per thread
-     *     int thread_data = ...
-     *
-     *     // Return the warp-wide reductions to each lane0
-     *     int warp_id = threadIdx.x / 32;
-     *     int aggregate = WarpReduce(temp_storage[warp_id]).Reduce(
-     *         thread_data, cub::Max());
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, 1, 2, 3, ..., 127}</tt>.
-     * The corresponding output \p aggregate in threads 0, 32, 64, and 96 will \p 31, \p 63,
-     * \p 95, and \p 127, respectively  (and is undefined in other threads).
-     *
-     * \tparam ReductionOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <typename ReductionOp>
-    __device__ __forceinline__ T Reduce(
-        T                   input,              ///< [in] Calling thread's input
-        ReductionOp         reduction_op)       ///< [in] Binary reduction operator
-    {
-        return InternalWarpReduce(temp_storage).template Reduce<true>(input, LOGICAL_WARP_THREADS, reduction_op);
-    }
-
-    /**
-     * \brief Computes a partially-full warp-wide reduction in the calling warp using the specified binary reduction functor.  The output is valid in warp <em>lane</em><sub>0</sub>.
-     *
-     * All threads across the calling warp must agree on the same value for \p valid_items.  Otherwise the result is undefined.
-     *
-     * Supports non-commutative reduction operators
-     *
-     * \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a max reduction within a single, partially-full
-     * block of 32 threads (one warp).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(int *d_data, int valid_items)
-     * {
-     *     // Specialize WarpReduce for type int
-     *     typedef cub::WarpReduce<int> WarpReduce;
-     *
-     *     // Allocate WarpReduce shared memory for one warp
-     *     __shared__ typename WarpReduce::TempStorage temp_storage;
-     *
-     *     // Obtain one input item per thread if in range
-     *     int thread_data;
-     *     if (threadIdx.x < valid_items)
-     *         thread_data = d_data[threadIdx.x];
-     *
-     *     // Return the warp-wide reductions to each lane0
-     *     int aggregate = WarpReduce(temp_storage).Reduce(
-     *         thread_data, cub::Max(), valid_items);
-     *
-     * \endcode
-     * \par
-     * Suppose the input \p d_data is <tt>{0, 1, 2, 3, 4, ...</tt> and \p valid_items
-     * is \p 4.  The corresponding output \p aggregate in thread0 is \p 3 (and is
-     * undefined in other threads).
-     *
-     * \tparam ReductionOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <typename ReductionOp>
-    __device__ __forceinline__ T Reduce(
-        T                   input,              ///< [in] Calling thread's input
-        ReductionOp         reduction_op,       ///< [in] Binary reduction operator
-        int                 valid_items)        ///< [in] Total number of valid items in the calling thread's logical warp (may be less than \p LOGICAL_WARP_THREADS)
-    {
-        return InternalWarpReduce(temp_storage).template Reduce<false>(input, valid_items, reduction_op);
-    }
-
-
-    /**
-     * \brief Computes a segmented reduction in the calling warp where segments are defined by head-flags.  The reduction of each segment is returned to the first lane in that segment (which always includes <em>lane</em><sub>0</sub>).
-     *
-     * Supports non-commutative reduction operators
-     *
-     * \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a head-segmented warp max
-     * reduction within a block of 32 threads (one warp).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpReduce for type int
-     *     typedef cub::WarpReduce<int> WarpReduce;
-     *
-     *     // Allocate WarpReduce shared memory for one warp
-     *     __shared__ typename WarpReduce::TempStorage temp_storage;
-     *
-     *     // Obtain one input item and flag per thread
-     *     int thread_data = ...
-     *     int head_flag = ...
-     *
-     *     // Return the warp-wide reductions to each lane0
-     *     int aggregate = WarpReduce(temp_storage).HeadSegmentedReduce(
-     *         thread_data, head_flag, cub::Max());
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data and \p head_flag across the block of threads
-     * is <tt>{0, 1, 2, 3, ..., 31</tt> and is <tt>{1, 0, 0, 0, 1, 0, 0, 0, ..., 1, 0, 0, 0</tt>,
-     * respectively.  The corresponding output \p aggregate in threads 0, 4, 8, etc. will be
-     * \p 3, \p 7, \p 11, etc. (and is undefined in other threads).
-     *
-     * \tparam ReductionOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <
-        typename            ReductionOp,
-        typename            FlagT>
-    __device__ __forceinline__ T HeadSegmentedReduce(
-        T                   input,              ///< [in] Calling thread's input
-        FlagT               head_flag,          ///< [in] Head flag denoting whether or not \p input is the start of a new segment
-        ReductionOp         reduction_op)       ///< [in] Reduction operator
-    {
-        return InternalWarpReduce(temp_storage).template SegmentedReduce<true>(input, head_flag, reduction_op);
-    }
-
-
-    /**
-     * \brief Computes a segmented reduction in the calling warp where segments are defined by tail-flags.  The reduction of each segment is returned to the first lane in that segment (which always includes <em>lane</em><sub>0</sub>).
-     *
-     * Supports non-commutative reduction operators
-     *
-     * \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates a tail-segmented warp max
-     * reduction within a block of 32 threads (one warp).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpReduce for type int
-     *     typedef cub::WarpReduce<int> WarpReduce;
-     *
-     *     // Allocate WarpReduce shared memory for one warp
-     *     __shared__ typename WarpReduce::TempStorage temp_storage;
-     *
-     *     // Obtain one input item and flag per thread
-     *     int thread_data = ...
-     *     int tail_flag = ...
-     *
-     *     // Return the warp-wide reductions to each lane0
-     *     int aggregate = WarpReduce(temp_storage).TailSegmentedReduce(
-     *         thread_data, tail_flag, cub::Max());
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data and \p tail_flag across the block of threads
-     * is <tt>{0, 1, 2, 3, ..., 31</tt> and is <tt>{0, 0, 0, 1, 0, 0, 0, 1, ..., 0, 0, 0, 1</tt>,
-     * respectively.  The corresponding output \p aggregate in threads 0, 4, 8, etc. will be
-     * \p 3, \p 7, \p 11, etc. (and is undefined in other threads).
-     *
-     * \tparam ReductionOp     <b>[inferred]</b> Binary reduction operator type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <
-        typename            ReductionOp,
-        typename            FlagT>
-    __device__ __forceinline__ T TailSegmentedReduce(
-        T                   input,              ///< [in] Calling thread's input
-        FlagT               tail_flag,          ///< [in] Tail flag denoting whether or not \p input is the end of the current segment
-        ReductionOp         reduction_op)       ///< [in] Reduction operator
-    {
-        return InternalWarpReduce(temp_storage).template SegmentedReduce<false>(input, tail_flag, reduction_op);
-    }
-
-
-
-    //@}  end member group
-};
-
-/** @} */       // end group WarpModule
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pfsimulator/third_party/cub/warp/warp_scan.cuh b/pfsimulator/third_party/cub/warp/warp_scan.cuh
deleted file mode 100644
index c7af0d343..000000000
--- a/pfsimulator/third_party/cub/warp/warp_scan.cuh
+++ /dev/null
@@ -1,936 +0,0 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
-
-/**
- * \file
- * The cub::WarpScan class provides [<em>collective</em>](index.html#sec0) methods for computing a parallel prefix scan of items partitioned across a CUDA thread warp.
- */
-
-#pragma once
-
-#include "specializations/warp_scan_shfl.cuh"
-#include "specializations/warp_scan_smem.cuh"
-#include "../thread/thread_operators.cuh"
-#include "../util_arch.cuh"
-#include "../util_type.cuh"
-#include "../util_namespace.cuh"
-
-/// Optional outer namespace(s)
-CUB_NS_PREFIX
-
-/// CUB namespace
-namespace cub {
-
-/**
- * \addtogroup WarpModule
- * @{
- */
-
-/**
- * \brief The WarpScan class provides [<em>collective</em>](index.html#sec0) methods for computing a parallel prefix scan of items partitioned across a CUDA thread warp.  ![](warp_scan_logo.png)
- *
- * \tparam T                        The scan input/output element type
- * \tparam LOGICAL_WARP_THREADS     <b>[optional]</b> The number of threads per "logical" warp (may be less than the number of hardware warp threads).  Default is the warp size associated with the CUDA Compute Capability targeted by the compiler (e.g., 32 threads for SM20).
- * \tparam PTX_ARCH                 <b>[optional]</b> \ptxversion
- *
- * \par Overview
- * - Given a list of input elements and a binary reduction operator, a [<em>prefix scan</em>](http://en.wikipedia.org/wiki/Prefix_sum)
- *   produces an output list where each element is computed to be the reduction
- *   of the elements occurring earlier in the input list.  <em>Prefix sum</em>
- *   connotes a prefix scan with the addition operator. The term \em inclusive indicates
- *   that the <em>i</em><sup>th</sup> output reduction incorporates the <em>i</em><sup>th</sup> input.
- *   The term \em exclusive indicates the <em>i</em><sup>th</sup> input is not incorporated into
- *   the <em>i</em><sup>th</sup> output reduction.
- * - Supports non-commutative scan operators
- * - Supports "logical" warps smaller than the physical warp size (e.g., a logical warp of 8 threads)
- * - The number of entrant threads must be an multiple of \p LOGICAL_WARP_THREADS
- *
- * \par Performance Considerations
- * - Uses special instructions when applicable (e.g., warp \p SHFL)
- * - Uses synchronization-free communication between warp lanes when applicable
- * - Incurs zero bank conflicts for most types
- * - Computation is slightly more efficient (i.e., having lower instruction overhead) for:
- *     - Summation (<b><em>vs.</em></b> generic scan)
- *     - The architecture's warp size is a whole multiple of \p LOGICAL_WARP_THREADS
- *
- * \par Simple Examples
- * \warpcollective{WarpScan}
- * \par
- * The code snippet below illustrates four concurrent warp prefix sums within a block of
- * 128 threads (one per each of the 32-thread warps).
- * \par
- * \code
- * #include <cub/cub.cuh>
- *
- * __global__ void ExampleKernel(...)
- * {
- *     // Specialize WarpScan for type int
- *     typedef cub::WarpScan<int> WarpScan;
- *
- *     // Allocate WarpScan shared memory for 4 warps
- *     __shared__ typename WarpScan::TempStorage temp_storage[4];
- *
- *     // Obtain one input item per thread
- *     int thread_data = ...
- *
- *     // Compute warp-wide prefix sums
- *     int warp_id = threadIdx.x / 32;
- *     WarpScan(temp_storage[warp_id]).ExclusiveSum(thread_data, thread_data);
- *
- * \endcode
- * \par
- * Suppose the set of input \p thread_data across the block of threads is <tt>{1, 1, 1, 1, ...}</tt>.
- * The corresponding output \p thread_data in each of the four warps of threads will be
- * <tt>0, 1, 2, 3, ..., 31}</tt>.
- *
- * \par
- * The code snippet below illustrates a single warp prefix sum within a block of
- * 128 threads.
- * \par
- * \code
- * #include <cub/cub.cuh>
- *
- * __global__ void ExampleKernel(...)
- * {
- *     // Specialize WarpScan for type int
- *     typedef cub::WarpScan<int> WarpScan;
- *
- *     // Allocate WarpScan shared memory for one warp
- *     __shared__ typename WarpScan::TempStorage temp_storage;
- *     ...
- *
- *     // Only the first warp performs a prefix sum
- *     if (threadIdx.x < 32)
- *     {
- *         // Obtain one input item per thread
- *         int thread_data = ...
- *
- *         // Compute warp-wide prefix sums
- *         WarpScan(temp_storage).ExclusiveSum(thread_data, thread_data);
- *
- * \endcode
- * \par
- * Suppose the set of input \p thread_data across the warp of threads is <tt>{1, 1, 1, 1, ...}</tt>.
- * The corresponding output \p thread_data will be <tt>{0, 1, 2, 3, ..., 31}</tt>.
- *
- */
-template <
-    typename    T,
-    int         LOGICAL_WARP_THREADS    = CUB_PTX_WARP_THREADS,
-    int         PTX_ARCH                = CUB_PTX_ARCH>
-class WarpScan
-{
-private:
-
-    /******************************************************************************
-     * Constants and type definitions
-     ******************************************************************************/
-
-    enum
-    {
-        /// Whether the logical warp size and the PTX warp size coincide
-        IS_ARCH_WARP = (LOGICAL_WARP_THREADS == CUB_WARP_THREADS(PTX_ARCH)),
-
-        /// Whether the logical warp size is a power-of-two
-        IS_POW_OF_TWO = ((LOGICAL_WARP_THREADS & (LOGICAL_WARP_THREADS - 1)) == 0),
-
-        /// Whether the data type is an integer (which has fully-associative addition)
-        IS_INTEGER = ((Traits<T>::CATEGORY == SIGNED_INTEGER) || (Traits<T>::CATEGORY == UNSIGNED_INTEGER))
-    };
-
-    /// Internal specialization.  Use SHFL-based scan if (architecture is >= SM30) and (LOGICAL_WARP_THREADS is a power-of-two)
-    typedef typename If<(PTX_ARCH >= 300) && (IS_POW_OF_TWO),
-        WarpScanShfl<T, LOGICAL_WARP_THREADS, PTX_ARCH>,
-        WarpScanSmem<T, LOGICAL_WARP_THREADS, PTX_ARCH> >::Type InternalWarpScan;
-
-    /// Shared memory storage layout type for WarpScan
-    typedef typename InternalWarpScan::TempStorage _TempStorage;
-
-
-    /******************************************************************************
-     * Thread fields
-     ******************************************************************************/
-
-    /// Shared storage reference
-    _TempStorage    &temp_storage;
-    unsigned int    lane_id;
-
-
-
-    /******************************************************************************
-     * Public types
-     ******************************************************************************/
-
-public:
-
-    /// \smemstorage{WarpScan}
-    struct TempStorage : Uninitialized<_TempStorage> {};
-
-
-    /******************************************************************//**
-     * \name Collective constructors
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Collective constructor using the specified memory allocation as temporary storage.  Logical warp and lane identifiers are constructed from <tt>threadIdx.x</tt>.
-     */
-    __device__ __forceinline__ WarpScan(
-        TempStorage &temp_storage)             ///< [in] Reference to memory allocation having layout type TempStorage
-    :
-        temp_storage(temp_storage.Alias()),
-        lane_id(IS_ARCH_WARP ?
-            LaneId() :
-            LaneId() % LOGICAL_WARP_THREADS)
-    {}
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Inclusive prefix sums
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Computes an inclusive prefix sum across the calling warp.
-     *
-     * \par
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates four concurrent warp-wide inclusive prefix sums within a block of
-     * 128 threads (one per each of the 32-thread warps).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpScan for type int
-     *     typedef cub::WarpScan<int> WarpScan;
-     *
-     *     // Allocate WarpScan shared memory for 4 warps
-     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
-     *
-     *     // Obtain one input item per thread
-     *     int thread_data = ...
-     *
-     *     // Compute inclusive warp-wide prefix sums
-     *     int warp_id = threadIdx.x / 32;
-     *     WarpScan(temp_storage[warp_id]).InclusiveSum(thread_data, thread_data);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{1, 1, 1, 1, ...}</tt>.
-     * The corresponding output \p thread_data in each of the four warps of threads will be
-     * <tt>1, 2, 3, ..., 32}</tt>.
-     */
-    __device__ __forceinline__ void InclusiveSum(
-        T               input,              ///< [in] Calling thread's input item.
-        T               &inclusive_output)  ///< [out] Calling thread's output item.  May be aliased with \p input.
-    {
-        InclusiveScan(input, inclusive_output, cub::Sum());
-    }
-
-
-    /**
-     * \brief Computes an inclusive prefix sum across the calling warp.  Also provides every thread with the warp-wide \p warp_aggregate of all inputs.
-     *
-     * \par
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates four concurrent warp-wide inclusive prefix sums within a block of
-     * 128 threads (one per each of the 32-thread warps).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpScan for type int
-     *     typedef cub::WarpScan<int> WarpScan;
-     *
-     *     // Allocate WarpScan shared memory for 4 warps
-     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
-     *
-     *     // Obtain one input item per thread
-     *     int thread_data = ...
-     *
-     *     // Compute inclusive warp-wide prefix sums
-     *     int warp_aggregate;
-     *     int warp_id = threadIdx.x / 32;
-     *     WarpScan(temp_storage[warp_id]).InclusiveSum(thread_data, thread_data, warp_aggregate);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{1, 1, 1, 1, ...}</tt>.
-     * The corresponding output \p thread_data in each of the four warps of threads will be
-     * <tt>1, 2, 3, ..., 32}</tt>.  Furthermore, \p warp_aggregate for all threads in all warps will be \p 32.
-     */
-    __device__ __forceinline__ void InclusiveSum(
-        T               input,              ///< [in] Calling thread's input item.
-        T               &inclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
-        T               &warp_aggregate)    ///< [out] Warp-wide aggregate reduction of input items.
-    {
-        InclusiveScan(input, inclusive_output, cub::Sum(), warp_aggregate);
-    }
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Exclusive prefix sums
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Computes an exclusive prefix sum across the calling warp.  The value of 0 is applied as the initial value, and is assigned to \p exclusive_output in <em>thread</em><sub>0</sub>.
-     *
-     * \par
-     *  - \identityzero
-     *  - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates four concurrent warp-wide exclusive prefix sums within a block of
-     * 128 threads (one per each of the 32-thread warps).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpScan for type int
-     *     typedef cub::WarpScan<int> WarpScan;
-     *
-     *     // Allocate WarpScan shared memory for 4 warps
-     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
-     *
-     *     // Obtain one input item per thread
-     *     int thread_data = ...
-     *
-     *     // Compute exclusive warp-wide prefix sums
-     *     int warp_id = threadIdx.x / 32;
-     *     WarpScan(temp_storage[warp_id]).ExclusiveSum(thread_data, thread_data);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{1, 1, 1, 1, ...}</tt>.
-     * The corresponding output \p thread_data in each of the four warps of threads will be
-     * <tt>0, 1, 2, ..., 31}</tt>.
-     *
-     */
-    __device__ __forceinline__ void ExclusiveSum(
-        T               input,              ///< [in] Calling thread's input item.
-        T               &exclusive_output)  ///< [out] Calling thread's output item.  May be aliased with \p input.
-    {
-        T initial_value = 0;
-        ExclusiveScan(input, exclusive_output, initial_value, cub::Sum());
-    }
-
-
-    /**
-     * \brief Computes an exclusive prefix sum across the calling warp.  The value of 0 is applied as the initial value, and is assigned to \p exclusive_output in <em>thread</em><sub>0</sub>.  Also provides every thread with the warp-wide \p warp_aggregate of all inputs.
-     *
-     * \par
-     *  - \identityzero
-     *  - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates four concurrent warp-wide exclusive prefix sums within a block of
-     * 128 threads (one per each of the 32-thread warps).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpScan for type int
-     *     typedef cub::WarpScan<int> WarpScan;
-     *
-     *     // Allocate WarpScan shared memory for 4 warps
-     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
-     *
-     *     // Obtain one input item per thread
-     *     int thread_data = ...
-     *
-     *     // Compute exclusive warp-wide prefix sums
-     *     int warp_aggregate;
-     *     int warp_id = threadIdx.x / 32;
-     *     WarpScan(temp_storage[warp_id]).ExclusiveSum(thread_data, thread_data, warp_aggregate);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{1, 1, 1, 1, ...}</tt>.
-     * The corresponding output \p thread_data in each of the four warps of threads will be
-     * <tt>0, 1, 2, ..., 31}</tt>.  Furthermore, \p warp_aggregate for all threads in all warps will be \p 32.
-     */
-    __device__ __forceinline__ void ExclusiveSum(
-        T               input,              ///< [in] Calling thread's input item.
-        T               &exclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
-        T               &warp_aggregate)    ///< [out] Warp-wide aggregate reduction of input items.
-    {
-        T initial_value = 0;
-        ExclusiveScan(input, exclusive_output, initial_value, cub::Sum(), warp_aggregate);
-    }
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Inclusive prefix scans
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Computes an inclusive prefix scan using the specified binary scan functor across the calling warp.
-     *
-     * \par
-     *  - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates four concurrent warp-wide inclusive prefix max scans within a block of
-     * 128 threads (one per each of the 32-thread warps).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpScan for type int
-     *     typedef cub::WarpScan<int> WarpScan;
-     *
-     *     // Allocate WarpScan shared memory for 4 warps
-     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
-     *
-     *     // Obtain one input item per thread
-     *     int thread_data = ...
-     *
-     *     // Compute inclusive warp-wide prefix max scans
-     *     int warp_id = threadIdx.x / 32;
-     *     WarpScan(temp_storage[warp_id]).InclusiveScan(thread_data, thread_data, cub::Max());
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, -1, 2, -3, ..., 126, -127}</tt>.
-     * The corresponding output \p thread_data in the first warp would be
-     * <tt>0, 0, 2, 2, ..., 30, 30</tt>, the output for the second warp would be <tt>32, 32, 34, 34, ..., 62, 62</tt>, etc.
-     *
-     * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <typename ScanOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T               input,              ///< [in] Calling thread's input item.
-        T               &inclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
-        ScanOp          scan_op)            ///< [in] Binary scan operator
-    {
-        InternalWarpScan(temp_storage).InclusiveScan(input, inclusive_output, scan_op);
-    }
-
-
-    /**
-     * \brief Computes an inclusive prefix scan using the specified binary scan functor across the calling warp.  Also provides every thread with the warp-wide \p warp_aggregate of all inputs.
-     *
-     * \par
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates four concurrent warp-wide inclusive prefix max scans within a block of
-     * 128 threads (one per each of the 32-thread warps).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpScan for type int
-     *     typedef cub::WarpScan<int> WarpScan;
-     *
-     *     // Allocate WarpScan shared memory for 4 warps
-     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
-     *
-     *     // Obtain one input item per thread
-     *     int thread_data = ...
-     *
-     *     // Compute inclusive warp-wide prefix max scans
-     *     int warp_aggregate;
-     *     int warp_id = threadIdx.x / 32;
-     *     WarpScan(temp_storage[warp_id]).InclusiveScan(
-     *         thread_data, thread_data, cub::Max(), warp_aggregate);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, -1, 2, -3, ..., 126, -127}</tt>.
-     * The corresponding output \p thread_data in the first warp would be
-     * <tt>0, 0, 2, 2, ..., 30, 30</tt>, the output for the second warp would be <tt>32, 32, 34, 34, ..., 62, 62</tt>, etc.
-     * Furthermore, \p warp_aggregate would be assigned \p 30 for threads in the first warp, \p 62 for threads
-     * in the second warp, etc.
-     *
-     * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <typename ScanOp>
-    __device__ __forceinline__ void InclusiveScan(
-        T               input,              ///< [in] Calling thread's input item.
-        T               &inclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
-        ScanOp          scan_op,            ///< [in] Binary scan operator
-        T               &warp_aggregate)    ///< [out] Warp-wide aggregate reduction of input items.
-    {
-        InternalWarpScan(temp_storage).InclusiveScan(input, inclusive_output, scan_op, warp_aggregate);
-    }
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Exclusive prefix scans
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Computes an exclusive prefix scan using the specified binary scan functor across the calling warp.  Because no initial value is supplied, the \p output computed for <em>warp-lane</em><sub>0</sub> is undefined.
-     *
-     * \par
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates four concurrent warp-wide exclusive prefix max scans within a block of
-     * 128 threads (one per each of the 32-thread warps).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpScan for type int
-     *     typedef cub::WarpScan<int> WarpScan;
-     *
-     *     // Allocate WarpScan shared memory for 4 warps
-     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
-     *
-     *     // Obtain one input item per thread
-     *     int thread_data = ...
-     *
-     *     // Compute exclusive warp-wide prefix max scans
-     *     int warp_id = threadIdx.x / 32;
-     *     WarpScan(temp_storage[warp_id]).ExclusiveScan(thread_data, thread_data, cub::Max());
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, -1, 2, -3, ..., 126, -127}</tt>.
-     * The corresponding output \p thread_data in the first warp would be
-     * <tt>?, 0, 0, 2, ..., 28, 30</tt>, the output for the second warp would be <tt>?, 32, 32, 34, ..., 60, 62</tt>, etc.
-     * (The output \p thread_data in warp lane<sub>0</sub> is undefined.)
-     *
-     * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,              ///< [in] Calling thread's input item.
-        T               &exclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
-        ScanOp          scan_op)            ///< [in] Binary scan operator
-    {
-        InternalWarpScan internal(temp_storage);
-
-        T inclusive_output;
-        internal.InclusiveScan(input, inclusive_output, scan_op);
-
-        internal.Update(
-            input,
-            inclusive_output,
-            exclusive_output,
-            scan_op,
-            Int2Type<IS_INTEGER>());
-    }
-
-
-    /**
-     * \brief Computes an exclusive prefix scan using the specified binary scan functor across the calling warp.
-     *
-     * \par
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates four concurrent warp-wide exclusive prefix max scans within a block of
-     * 128 threads (one per each of the 32-thread warps).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpScan for type int
-     *     typedef cub::WarpScan<int> WarpScan;
-     *
-     *     // Allocate WarpScan shared memory for 4 warps
-     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
-     *
-     *     // Obtain one input item per thread
-     *     int thread_data = ...
-     *
-     *     // Compute exclusive warp-wide prefix max scans
-     *     int warp_id = threadIdx.x / 32;
-     *     WarpScan(temp_storage[warp_id]).ExclusiveScan(thread_data, thread_data, INT_MIN, cub::Max());
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, -1, 2, -3, ..., 126, -127}</tt>.
-     * The corresponding output \p thread_data in the first warp would be
-     * <tt>INT_MIN, 0, 0, 2, ..., 28, 30</tt>, the output for the second warp would be <tt>30, 32, 32, 34, ..., 60, 62</tt>, etc.
-     *
-     * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,              ///< [in] Calling thread's input item.
-        T               &exclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
-        T               initial_value,      ///< [in] Initial value to seed the exclusive scan
-        ScanOp          scan_op)            ///< [in] Binary scan operator
-    {
-        InternalWarpScan internal(temp_storage);
-
-        T inclusive_output;
-        internal.InclusiveScan(input, inclusive_output, scan_op);
-
-        internal.Update(
-            input,
-            inclusive_output,
-            exclusive_output,
-            scan_op,
-            initial_value,
-            Int2Type<IS_INTEGER>());
-    }
-
-
-    /**
-     * \brief Computes an exclusive prefix scan using the specified binary scan functor across the calling warp.  Because no initial value is supplied, the \p output computed for <em>warp-lane</em><sub>0</sub> is undefined.  Also provides every thread with the warp-wide \p warp_aggregate of all inputs.
-     *
-     * \par
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates four concurrent warp-wide exclusive prefix max scans within a block of
-     * 128 threads (one per each of the 32-thread warps).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpScan for type int
-     *     typedef cub::WarpScan<int> WarpScan;
-     *
-     *     // Allocate WarpScan shared memory for 4 warps
-     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
-     *
-     *     // Obtain one input item per thread
-     *     int thread_data = ...
-     *
-     *     // Compute exclusive warp-wide prefix max scans
-     *     int warp_aggregate;
-     *     int warp_id = threadIdx.x / 32;
-     *     WarpScan(temp_storage[warp_id]).ExclusiveScan(thread_data, thread_data, cub::Max(), warp_aggregate);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, -1, 2, -3, ..., 126, -127}</tt>.
-     * The corresponding output \p thread_data in the first warp would be
-     * <tt>?, 0, 0, 2, ..., 28, 30</tt>, the output for the second warp would be <tt>?, 32, 32, 34, ..., 60, 62</tt>, etc.
-     * (The output \p thread_data in warp lane<sub>0</sub> is undefined.)  Furthermore, \p warp_aggregate would be assigned \p 30 for threads in the first warp, \p 62 for threads
-     * in the second warp, etc.
-     *
-     * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,              ///< [in] Calling thread's input item.
-        T               &exclusive_output,   ///< [out] Calling thread's output item.  May be aliased with \p input.
-        ScanOp          scan_op,            ///< [in] Binary scan operator
-        T               &warp_aggregate)    ///< [out] Warp-wide aggregate reduction of input items.
-    {
-        InternalWarpScan internal(temp_storage);
-
-        T inclusive_output;
-        internal.InclusiveScan(input, inclusive_output, scan_op);
-
-        internal.Update(
-            input,
-            inclusive_output,
-            exclusive_output,
-            warp_aggregate,
-            scan_op,
-            Int2Type<IS_INTEGER>());
-    }
-
-
-    /**
-     * \brief Computes an exclusive prefix scan using the specified binary scan functor across the calling warp.  Also provides every thread with the warp-wide \p warp_aggregate of all inputs.
-     *
-     * \par
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates four concurrent warp-wide exclusive prefix max scans within a block of
-     * 128 threads (one per each of the 32-thread warps).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpScan for type int
-     *     typedef cub::WarpScan<int> WarpScan;
-     *
-     *     // Allocate WarpScan shared memory for 4 warps
-     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
-     *
-     *     // Obtain one input item per thread
-     *     int thread_data = ...
-     *
-     *     // Compute exclusive warp-wide prefix max scans
-     *     int warp_aggregate;
-     *     int warp_id = threadIdx.x / 32;
-     *     WarpScan(temp_storage[warp_id]).ExclusiveScan(thread_data, thread_data, INT_MIN, cub::Max(), warp_aggregate);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, -1, 2, -3, ..., 126, -127}</tt>.
-     * The corresponding output \p thread_data in the first warp would be
-     * <tt>INT_MIN, 0, 0, 2, ..., 28, 30</tt>, the output for the second warp would be <tt>30, 32, 32, 34, ..., 60, 62</tt>, etc.
-     * Furthermore, \p warp_aggregate would be assigned \p 30 for threads in the first warp, \p 62 for threads
-     * in the second warp, etc.
-     *
-     * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <typename ScanOp>
-    __device__ __forceinline__ void ExclusiveScan(
-        T               input,              ///< [in] Calling thread's input item.
-        T               &exclusive_output,  ///< [out] Calling thread's output item.  May be aliased with \p input.
-        T               initial_value,      ///< [in] Initial value to seed the exclusive scan
-        ScanOp          scan_op,            ///< [in] Binary scan operator
-        T               &warp_aggregate)    ///< [out] Warp-wide aggregate reduction of input items.
-    {
-        InternalWarpScan internal(temp_storage);
-
-        T inclusive_output;
-        internal.InclusiveScan(input, inclusive_output, scan_op);
-
-        internal.Update(
-            input,
-            inclusive_output,
-            exclusive_output,
-            warp_aggregate,
-            scan_op,
-            initial_value,
-            Int2Type<IS_INTEGER>());
-    }
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Combination (inclusive & exclusive) prefix scans
-     *********************************************************************/
-    //@{
-
-
-    /**
-     * \brief Computes both inclusive and exclusive prefix scans using the specified binary scan functor across the calling warp.  Because no initial value is supplied, the \p exclusive_output computed for <em>warp-lane</em><sub>0</sub> is undefined.
-     *
-     * \par
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates four concurrent warp-wide exclusive prefix max scans within a block of
-     * 128 threads (one per each of the 32-thread warps).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpScan for type int
-     *     typedef cub::WarpScan<int> WarpScan;
-     *
-     *     // Allocate WarpScan shared memory for 4 warps
-     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
-     *
-     *     // Obtain one input item per thread
-     *     int thread_data = ...
-     *
-     *     // Compute exclusive warp-wide prefix max scans
-     *     int inclusive_partial, exclusive_partial;
-     *     WarpScan(temp_storage[warp_id]).Scan(thread_data, inclusive_partial, exclusive_partial, cub::Max());
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, -1, 2, -3, ..., 126, -127}</tt>.
-     * The corresponding output \p inclusive_partial in the first warp would be
-     * <tt>0, 0, 2, 2, ..., 30, 30</tt>, the output for the second warp would be <tt>32, 32, 34, 34, ..., 62, 62</tt>, etc.
-     * The corresponding output \p exclusive_partial in the first warp would be
-     * <tt>?, 0, 0, 2, ..., 28, 30</tt>, the output for the second warp would be <tt>?, 32, 32, 34, ..., 60, 62</tt>, etc.
-     * (The output \p thread_data in warp lane<sub>0</sub> is undefined.)
-     *
-     * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <typename ScanOp>
-    __device__ __forceinline__ void Scan(
-        T               input,              ///< [in] Calling thread's input item.
-        T               &inclusive_output,  ///< [out] Calling thread's inclusive-scan output item.
-        T               &exclusive_output,  ///< [out] Calling thread's exclusive-scan output item.
-        ScanOp          scan_op)            ///< [in] Binary scan operator
-    {
-        InternalWarpScan internal(temp_storage);
-
-        internal.InclusiveScan(input, inclusive_output, scan_op);
-
-        internal.Update(
-            input,
-            inclusive_output,
-            exclusive_output,
-            scan_op,
-            Int2Type<IS_INTEGER>());
-    }
-
-
-    /**
-     * \brief Computes both inclusive and exclusive prefix scans using the specified binary scan functor across the calling warp.
-     *
-     * \par
-     *  - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates four concurrent warp-wide prefix max scans within a block of
-     * 128 threads (one per each of the 32-thread warps).
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpScan for type int
-     *     typedef cub::WarpScan<int> WarpScan;
-     *
-     *     // Allocate WarpScan shared memory for 4 warps
-     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
-     *
-     *     // Obtain one input item per thread
-     *     int thread_data = ...
-     *
-     *     // Compute inclusive warp-wide prefix max scans
-     *     int warp_id = threadIdx.x / 32;
-     *     int inclusive_partial, exclusive_partial;
-     *     WarpScan(temp_storage[warp_id]).Scan(thread_data, inclusive_partial, exclusive_partial, INT_MIN, cub::Max());
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, -1, 2, -3, ..., 126, -127}</tt>.
-     * The corresponding output \p inclusive_partial in the first warp would be
-     * <tt>0, 0, 2, 2, ..., 30, 30</tt>, the output for the second warp would be <tt>32, 32, 34, 34, ..., 62, 62</tt>, etc.
-     * The corresponding output \p exclusive_partial in the first warp would be
-     * <tt>INT_MIN, 0, 0, 2, ..., 28, 30</tt>, the output for the second warp would be <tt>30, 32, 32, 34, ..., 60, 62</tt>, etc.
-     *
-     * \tparam ScanOp     <b>[inferred]</b> Binary scan operator type having member <tt>T operator()(const T &a, const T &b)</tt>
-     */
-    template <typename ScanOp>
-    __device__ __forceinline__ void Scan(
-        T               input,              ///< [in] Calling thread's input item.
-        T               &inclusive_output,  ///< [out] Calling thread's inclusive-scan output item.
-        T               &exclusive_output,  ///< [out] Calling thread's exclusive-scan output item.
-        T               initial_value,      ///< [in] Initial value to seed the exclusive scan
-        ScanOp          scan_op)            ///< [in] Binary scan operator
-    {
-        InternalWarpScan internal(temp_storage);
-
-        internal.InclusiveScan(input, inclusive_output, scan_op);
-
-        internal.Update(
-            input,
-            inclusive_output,
-            exclusive_output,
-            scan_op,
-            initial_value,
-            Int2Type<IS_INTEGER>());
-    }
-
-
-
-    //@}  end member group
-    /******************************************************************//**
-     * \name Data exchange
-     *********************************************************************/
-    //@{
-
-    /**
-     * \brief Broadcast the value \p input from <em>warp-lane</em><sub><tt>src_lane</tt></sub> to all lanes in the warp
-     *
-     * \par
-     * - \smemreuse
-     *
-     * \par Snippet
-     * The code snippet below illustrates the warp-wide broadcasts of values from
-     * lanes<sub>0</sub> in each of four warps to all other threads in those warps.
-     * \par
-     * \code
-     * #include <cub/cub.cuh>
-     *
-     * __global__ void ExampleKernel(...)
-     * {
-     *     // Specialize WarpScan for type int
-     *     typedef cub::WarpScan<int> WarpScan;
-     *
-     *     // Allocate WarpScan shared memory for 4 warps
-     *     __shared__ typename WarpScan::TempStorage temp_storage[4];
-     *
-     *     // Obtain one input item per thread
-     *     int thread_data = ...
-     *
-     *     // Broadcast from lane0 in each warp to all other threads in the warp
-     *     int warp_id = threadIdx.x / 32;
-     *     thread_data = WarpScan(temp_storage[warp_id]).Broadcast(thread_data, 0);
-     *
-     * \endcode
-     * \par
-     * Suppose the set of input \p thread_data across the block of threads is <tt>{0, 1, 2, 3, ..., 127}</tt>.
-     * The corresponding output \p thread_data will be
-     * <tt>{0, 0, ..., 0}</tt> in warp<sub>0</sub>,
-     * <tt>{32, 32, ..., 32}</tt> in warp<sub>1</sub>,
-     * <tt>{64, 64, ..., 64}</tt> in warp<sub>2</sub>, etc.
-     */
-    __device__ __forceinline__ T Broadcast(
-        T               input,              ///< [in] The value to broadcast
-        unsigned int    src_lane)           ///< [in] Which warp lane is to do the broadcasting
-    {
-        return InternalWarpScan(temp_storage).Broadcast(input, src_lane);
-    }
-
-    //@}  end member group
-
-};
-
-/** @} */       // end group WarpModule
-
-}               // CUB namespace
-CUB_NS_POSTFIX  // Optional outer namespace(s)
diff --git a/pftools/axpy.c b/pftools/axpy.c
index c607fbaf2..c704af418 100644
--- a/pftools/axpy.c
+++ b/pftools/axpy.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Axpy
 *
diff --git a/pftools/bgmsfem2pfsol.c b/pftools/bgmsfem2pfsol.c
index 13376a8ae..ce9b92380 100644
--- a/pftools/bgmsfem2pfsol.c
+++ b/pftools/bgmsfem2pfsol.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <stdio.h>
 #include <stdlib.h>
diff --git a/pftools/chunk/code/chunk.f.in b/pftools/chunk/code/chunk.f.in
index 146b3d0a3..826237175 100644
--- a/pftools/chunk/code/chunk.f.in
+++ b/pftools/chunk/code/chunk.f.in
@@ -185,7 +185,7 @@ c     check if background is white
 c
        iwhite=0
        if(brgb(1).eq.1..and.brgb(2).eq.1..and.brgb(3).eq.1.)iwhite=1
-       iwhite = 1
+       iwhite = 1
 	 if(iwhite.eq.0)then
          write(lpsout,924) (brgb(i),i=1,3) 
  924     format(3f8.3,' rgb')
@@ -1393,7 +1393,7 @@ c
         gticmin=dgtic*nint(xlmin/dgtic) 
         if(gticmin.lt.xlmin)gticmin=gticmin+dgtic
         glegmin=dleglab*nint(xlmin/dleglab) 
-        if(glegmin.lt.xlmin)glegmin=glegmin+dleglab
+        if(glegmin.lt.xlmin)glegmin=glegmin+dleglab
 	print*, glegmin, xlmin, dleglab, ilog
 c
 c  draw tics 
@@ -1437,7 +1437,7 @@ c     write label
           if(lego.eq.1)xt=xt-tic
           yt=y(2)-text_size
           if(lego.eq.1)yt=y(2)+tic*1.3
-         if(ilog.ne.1)then
+         if(ilog.ne.1)then
 	print*, lego, gleg, xt, yt
           if(lego.ne.1)call num2text(gleg,xt,yt,text_size,2,0.,1,ndec) 
           if(lego.eq.1)call num2text(gleg,xt,yt,text_size,2,-90.,2,ndec) 
@@ -1765,10 +1765,10 @@ c  do decimal places
        if(ndec.gt.0)then
          ibeg=ibeg+1
          cval(ibeg:ibeg)='.'
-         do 20 idec=-1,-ndec,-1
-	print*, vrem, vabs, 10.**float(idec), idec, val
-	vrem = 0.
-	!if (10.**float(idec) /= 0.)  
+         do 20 idec=-1,-ndec,-1
+	print*, vrem, vabs, 10.**float(idec), idec, val
+	vrem = 0.
+	!if (10.**float(idec) /= 0.)  
 	          vrem=amod(vabs,10.**float(idec))
            ival=nint((vabs-vrem)*10.**(-idec))
            if(idec.eq.-ndec)ival=nint(vabs*10.**float(-idec))
@@ -2209,7 +2209,7 @@ c
        if(iflag.eq.1)icolor=1
        iline=1
        if(lin_form.eq.1.or.lin_form.eq.3) then 
-          read(15,*) xi(1),yi(1),zi(1),wi(1),vi(1)
+          read(15,*) xi(1),yi(1),zi(1),wi(1),vi(1)
 	print*, xi(1),yi(1),zi(1),wi(1),vi(1)
        endif
        if(lin_form.eq.4)then
@@ -2232,8 +2232,8 @@ c
         wi(1)=wi(j)
         vi(1)=vi(j)
         do 950 j=2,100000
-          read(15,*,end=969) xi(j),yi(j),zi(j),wi(j),vi(j)
-
+          read(15,*,end=969) xi(j),yi(j),zi(j),wi(j),vi(j)
+
 
           vi(j)=vi(j)*scalef
           if(vi(j).eq.0.000) go to 959
@@ -2246,7 +2246,7 @@ c
          read(15,*,end=99) npoints
          init_color=0
          do 951 j=1,npoints
-           read(15,*,end=99) xi(j),yi(j),zi(j),wi(j),vi(j)
+           read(15,*,end=99) xi(j),yi(j),zi(j),wi(j),vi(j)
 	
            vi(j)=vi(j)*scalef
  951     continue
@@ -2257,7 +2257,7 @@ c
          wi(1)=wi(j)
          vi(1)=vi(j)
          do 952 j=2,100000
-           read(15,*,end=989) xi(j),yi(j),zi(j),wi(j),vi(j)
+           read(15,*,end=989) xi(j),yi(j),zi(j),wi(j),vi(j)
 	
            if(wi(j).ne.wi(j-1)) go to 979
            vi(j)=vi(j)*scalef
@@ -4351,4 +4351,4 @@ c    dark inner edge
  500   continue
  995   continue
        return
-       end
+       end
\ No newline at end of file
diff --git a/pftools/compute_domain.c b/pftools/compute_domain.c
index 037056a2f..a90934e3d 100644
--- a/pftools/compute_domain.c
+++ b/pftools/compute_domain.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <stdio.h>
 #include <math.h>
 
diff --git a/pftools/compute_domain.h b/pftools/compute_domain.h
index 2b9d18487..fd83087e1 100644
--- a/pftools/compute_domain.h
+++ b/pftools/compute_domain.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #ifndef COMPUTE_DOMAIN_HEADER
 #define COMPUTE_DOMAIN_HEADER
 
@@ -55,5 +55,6 @@ SubgridArray  *Extract2DDomain(
 }
 #endif
 
+
 #endif
 
diff --git a/pftools/databox.c b/pftools/databox.c
index bcb052232..05edac323 100644
--- a/pftools/databox.c
+++ b/pftools/databox.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * NewDatabox and FreeDatabox
 *
diff --git a/pftools/databox.h b/pftools/databox.h
index 52d980f62..ccf56ad8a 100644
--- a/pftools/databox.h
+++ b/pftools/databox.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Databox header file
 *
@@ -90,9 +90,9 @@ typedef struct {
 
 #define DataboxLabel(databox)   ((databox)->label)
 
-#define DataboxCoeff(databox, i, j, k) \
-  (DataboxCoeffs(databox) +            \
-   (k) * DataboxNy(databox) * DataboxNx(databox) + (j) * DataboxNx(databox) + (i))
+#define DataboxCoeff(databox, i, j, k)       \
+        (DataboxCoeffs(databox) +            \
+         (k) * DataboxNy(databox) * DataboxNx(databox) + (j) * DataboxNx(databox) + (i))
 
 
 /* Defines how a grid definition is */
diff --git a/pftools/diff.c b/pftools/diff.c
index 7d36c4789..210822190 100644
--- a/pftools/diff.c
+++ b/pftools/diff.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *
 * (C) 1993 Regents of the University of California.
diff --git a/pftools/diff.h b/pftools/diff.h
index a6768df90..92d23dca2 100644
--- a/pftools/diff.h
+++ b/pftools/diff.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /****************************************************************************
  * Header file for `diff.c'
  *
diff --git a/pftools/dirt.tcl b/pftools/dirt.tcl
index 1d06e4429..7f4ad75a0 100644
--- a/pftools/dirt.tcl
+++ b/pftools/dirt.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/enlargebox.c b/pftools/enlargebox.c
index 6c087ca4a..c1db6504f 100644
--- a/pftools/enlargebox.c
+++ b/pftools/enlargebox.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <stdlib.h>
 #include "enlargebox.h"
 
diff --git a/pftools/enlargebox.h b/pftools/enlargebox.h
index 218df4fca..ff982b94a 100644
--- a/pftools/enlargebox.h
+++ b/pftools/enlargebox.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef ENLARGE_HEADER
 #define ENLARGE_HEADER
diff --git a/pftools/error.c b/pftools/error.c
index 8422d0e6b..2ce055cb4 100644
--- a/pftools/error.c
+++ b/pftools/error.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Error
 *
diff --git a/pftools/error.h b/pftools/error.h
index fe128c3e0..d71c95832 100644
--- a/pftools/error.h
+++ b/pftools/error.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Error header file
 *
diff --git a/pftools/extrap_TIN.c b/pftools/extrap_TIN.c
index 9c189d78b..39a39ef83 100644
--- a/pftools/extrap_TIN.c
+++ b/pftools/extrap_TIN.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /****************************************************************
 * File: extrap_TIN.c
 *
@@ -74,8 +74,8 @@ char **argv;
   char         *end_of_file;
 
   float        *x_new,
-    *y_new,
-    *z_new;
+               *y_new,
+               *z_new;
 
   float A[9];
   float b[3];
diff --git a/pftools/file.c b/pftools/file.c
index a77f46d25..5f899cfcf 100644
--- a/pftools/file.c
+++ b/pftools/file.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <string.h>
 #include <stdlib.h>
 
diff --git a/pftools/file.h b/pftools/file.h
index 903aa146a..e6f88ba70 100644
--- a/pftools/file.h
+++ b/pftools/file.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #ifndef FILE_HEADER
 #define FILE_HEADER
 
diff --git a/pftools/flux.c b/pftools/flux.c
index 16f48caf3..f2bff91ee 100644
--- a/pftools/flux.c
+++ b/pftools/flux.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * CompFlux
 *
@@ -42,10 +42,10 @@
 #include "flux.h"
 
 #if 0
-#define Mean(a, b) (0.5*((a) + (b)))
+#define Mean(a, b) (0.5 * ((a) + (b)))
 #define Mean(a, b) (sqrt((a) * (b)))
 #endif
-#define Mean(a, b)    (((a) + (b)) ? ((2.0*(a)*(b)) / ((a) + (b))) : 0)
+#define Mean(a, b)    (((a) + (b)) ? ((2.0 * (a) * (b)) / ((a) + (b))) : 0)
 
 /*-----------------------------------------------------------------------
  * Compute net cell flux from conductivity and hydraulic head
@@ -65,9 +65,9 @@ Databox       *CompFlux(
 
   double qxp, qxm, qyp, qym, qzp, qzm;
   int cell,
-    cell_xm1, cell_xp1,
-    cell_ym1, cell_yp1,
-    cell_zm1, cell_zp1;
+      cell_xm1, cell_xp1,
+      cell_ym1, cell_yp1,
+      cell_zm1, cell_zp1;
   int ii, jj, kk;
 
   nx = DataboxNx(k);
diff --git a/pftools/flux.h b/pftools/flux.h
index 231e34f64..54b09655e 100644
--- a/pftools/flux.h
+++ b/pftools/flux.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Header file for `flux.c'
 *
diff --git a/pftools/fsb.tcl b/pftools/fsb.tcl
index 502a06cd8..2a2affafc 100644
--- a/pftools/fsb.tcl
+++ b/pftools/fsb.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/general.h b/pftools/general.h
index 5de96c1ce..9f623e48b 100644
--- a/pftools/general.h
+++ b/pftools/general.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef _GENERAL_HEADER
 #define _GENERAL_HEADER
@@ -41,10 +41,10 @@ extern "C" {
  *--------------------------------------------------------------------------*/
 
 #define talloc(type, count) \
-  ((count > 0) ? (type*)malloc((unsigned int)(sizeof(type) * (count))) : NULL)
+        ((count > 0) ? (type*)malloc((unsigned int)(sizeof(type) * (count))) : NULL)
 
 #define ctalloc(type, count) \
-  ((count > 0) ? (type*)calloc((unsigned int)(count), (unsigned int)sizeof(type)) : NULL)
+        ((count > 0) ? (type*)calloc((unsigned int)(count), (unsigned int)sizeof(type)) : NULL)
 
 /* note: the `else' is required to guarantee termination of the `if' */
 #define tfree(ptr) if (ptr) free(ptr); else
diff --git a/pftools/geom_sort_vertices.c b/pftools/geom_sort_vertices.c
index 2cd288a71..1a40abba0 100644
--- a/pftools/geom_sort_vertices.c
+++ b/pftools/geom_sort_vertices.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "general.h"
 #include "geometry.h"
@@ -38,35 +38,35 @@ typedef Vertex *EltType;
 
 #define QSORT_CROSSOVER 10
 
-#define Compare(result, vertex0, vertex1, compare_op) \
-  {                                                   \
-    result = 0;                                       \
-    if ((vertex0->z)compare_op(vertex1->z))           \
-      result = 1;                                     \
-    else if ((vertex0->z) == (vertex1->z))            \
-    {                                                 \
-      if ((vertex0->y)compare_op(vertex1->y))         \
-        result = 1;                                   \
-      else if ((vertex0->y) == (vertex1->y))          \
-      {                                               \
-        if ((vertex0->x)compare_op(vertex1->x))       \
-          result = 1;                                 \
-      }                                               \
-    }                                                 \
-  }
+#define Compare(result, vertex0, vertex1, compare_op)       \
+        {                                                   \
+          result = 0;                                       \
+          if ((vertex0->z)compare_op(vertex1->z))           \
+          result = 1;                                       \
+          else if ((vertex0->z) == (vertex1->z))            \
+          {                                                 \
+            if ((vertex0->y)compare_op(vertex1->y))         \
+            result = 1;                                     \
+            else if ((vertex0->y) == (vertex1->y))          \
+            {                                               \
+              if ((vertex0->x)compare_op(vertex1->x))       \
+              result = 1;                                   \
+            }                                               \
+          }                                                 \
+        }
 
 #define CompareLessThan(result, vertex0, vertex1) \
-  Compare(result, vertex0, vertex1, <)
+        Compare(result, vertex0, vertex1, <)
 
 #define CompareGreaterThan(result, vertex0, vertex1) \
-  Compare(result, vertex0, vertex1, >)
-
-#define Swap(array, i, j, tmp) \
-  {                            \
-    tmp = array[i];            \
-    array[i] = array[j];       \
-    array[j] = tmp;            \
-  }
+        Compare(result, vertex0, vertex1, >)
+
+#define Swap(array, i, j, tmp)       \
+        {                            \
+          tmp = array[i];            \
+          array[i] = array[j];       \
+          array[j] = tmp;            \
+        }
 
 
 /*--------------------------------------------------------------------------
diff --git a/pftools/geom_sort_xy_vertices.c b/pftools/geom_sort_xy_vertices.c
index 040aac1de..6912d1748 100644
--- a/pftools/geom_sort_xy_vertices.c
+++ b/pftools/geom_sort_xy_vertices.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "general.h"
 #include "geometry.h"
@@ -38,30 +38,30 @@ typedef Vertex *EltType;
 
 #define QSORT_CROSSOVER 10
 
-#define Compare(result, vertex0, vertex1, compare_op) \
-  {                                                   \
-    result = 0;                                       \
-    if ((vertex0->y)compare_op(vertex1->y))           \
-      result = 1;                                     \
-    else if ((vertex0->y) == (vertex1->y))            \
-    {                                                 \
-      if ((vertex0->x)compare_op(vertex1->x))         \
-        result = 1;                                   \
-    }                                                 \
-  }
+#define Compare(result, vertex0, vertex1, compare_op)       \
+        {                                                   \
+          result = 0;                                       \
+          if ((vertex0->y)compare_op(vertex1->y))           \
+          result = 1;                                       \
+          else if ((vertex0->y) == (vertex1->y))            \
+          {                                                 \
+            if ((vertex0->x)compare_op(vertex1->x))         \
+            result = 1;                                     \
+          }                                                 \
+        }
 
 #define CompareLessThan(result, vertex0, vertex1) \
-  Compare(result, vertex0, vertex1, <)
+        Compare(result, vertex0, vertex1, <)
 
 #define CompareGreaterThan(result, vertex0, vertex1) \
-  Compare(result, vertex0, vertex1, >)
-
-#define Swap(array, i, j, tmp) \
-  {                            \
-    tmp = array[i];            \
-    array[i] = array[j];       \
-    array[j] = tmp;            \
-  }
+        Compare(result, vertex0, vertex1, >)
+
+#define Swap(array, i, j, tmp)       \
+        {                            \
+          tmp = array[i];            \
+          array[i] = array[j];       \
+          array[j] = tmp;            \
+        }
 
 
 /*--------------------------------------------------------------------------
diff --git a/pftools/geometry.h b/pftools/geometry.h
index 4ab4f2f35..a55b392cb 100644
--- a/pftools/geometry.h
+++ b/pftools/geometry.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef _GEOMETRY_HEADER
 #define _GEOMETRY_HEADER
diff --git a/pftools/getsubbox.c b/pftools/getsubbox.c
index c5e943456..2b7acfa61 100644
--- a/pftools/getsubbox.c
+++ b/pftools/getsubbox.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 *  CompSub_Box
 *
diff --git a/pftools/gms.h b/pftools/gms.h
index c2b88c2e7..b831d098d 100644
--- a/pftools/gms.h
+++ b/pftools/gms.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef _GMS_HEADER
 #define _GMS_HEADER
diff --git a/pftools/gmsSOL2pfsol.c b/pftools/gmsSOL2pfsol.c
index 98240facf..85338f34e 100644
--- a/pftools/gmsSOL2pfsol.c
+++ b/pftools/gmsSOL2pfsol.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "gms.h"
 
diff --git a/pftools/gmsTINvertices.c b/pftools/gmsTINvertices.c
index 1f91701b9..41c0c3723 100644
--- a/pftools/gmsTINvertices.c
+++ b/pftools/gmsTINvertices.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "gms.h"
 
diff --git a/pftools/gmsfem2pfsol.sh b/pftools/gmsfem2pfsol.sh
index 412a45402..b582402e5 100644
--- a/pftools/gmsfem2pfsol.sh
+++ b/pftools/gmsfem2pfsol.sh
@@ -1,7 +1,7 @@
 #! /bin/sh
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/gmsreadSOL.c b/pftools/gmsreadSOL.c
index 9ce27847b..5330e896d 100644
--- a/pftools/gmsreadSOL.c
+++ b/pftools/gmsreadSOL.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "gms.h"
 
diff --git a/pftools/gmsreadTIN.c b/pftools/gmsreadTIN.c
index 63c304c04..259f1ee33 100644
--- a/pftools/gmsreadTIN.c
+++ b/pftools/gmsreadTIN.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "gms.h"
 
diff --git a/pftools/gmstriangulate.c b/pftools/gmstriangulate.c
index ee6bcec08..df37c565d 100644
--- a/pftools/gmstriangulate.c
+++ b/pftools/gmstriangulate.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "gms.h"
 
@@ -34,21 +34,21 @@
  * Compare macro
  *--------------------------------------------------------------------------*/
 
-#define Compare(result, vertex0, vertex1)   \
-  {                                         \
-    result = 0;                             \
-    if ((vertex0->y) < (vertex1->y))        \
-      result = -1;                          \
-    else if ((vertex0->y) > (vertex1->y))   \
-      result = 1;                           \
-    else                                    \
-    {                                       \
-      if ((vertex0->x) < (vertex1->x))      \
-        result = -1;                        \
-      else if ((vertex0->x) > (vertex1->x)) \
-        result = 1;                         \
-    }                                       \
-  }
+#define Compare(result, vertex0, vertex1)         \
+        {                                         \
+          result = 0;                             \
+          if ((vertex0->y) < (vertex1->y))        \
+          result = -1;                            \
+          else if ((vertex0->y) > (vertex1->y))   \
+          result = 1;                             \
+          else                                    \
+          {                                       \
+            if ((vertex0->x) < (vertex1->x))      \
+            result = -1;                          \
+            else if ((vertex0->x) > (vertex1->x)) \
+            result = 1;                           \
+          }                                       \
+        }
 
 
 /*--------------------------------------------------------------------------
diff --git a/pftools/gmswriteTIN.c b/pftools/gmswriteTIN.c
index 639eb1286..26637cd97 100644
--- a/pftools/gmswriteTIN.c
+++ b/pftools/gmswriteTIN.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "gms.h"
 
diff --git a/pftools/grid.c b/pftools/grid.c
index cbca97499..ad5e900db 100644
--- a/pftools/grid.c
+++ b/pftools/grid.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <math.h>
 #include "pfload_file.h"
diff --git a/pftools/grid.h b/pftools/grid.h
index 05014ce28..b35c51d5c 100644
--- a/pftools/grid.h
+++ b/pftools/grid.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #ifndef _GRID_HEADER
 #define _GRID_HEADER
 
@@ -97,7 +97,7 @@ typedef struct {
  *--------------------------------------------------------------------------*/
 
 #define SubgridArraySubgrid(subgrid_array, i) \
-  ((Subgrid*)SubregionArraySubregion(subgrid_array, i))
+        ((Subgrid*)SubregionArraySubregion(subgrid_array, i))
 #define SubgridArraySize(subgrid_array)  SubregionArraySize(subgrid_array)
 
 /*--------------------------------------------------------------------------
@@ -124,21 +124,21 @@ typedef struct {
  *--------------------------------------------------------------------------*/
 
 #define NewSubgrid(x, y, z, nx, ny, nz, rx, ry, rz, process) \
-  ((Subgrid*)NewSubregion(x, y, z, nx, ny, nz, 1, 1, 1, rx, ry, rz, process))
+        ((Subgrid*)NewSubregion(x, y, z, nx, ny, nz, 1, 1, 1, rx, ry, rz, process))
 
 #define NewSubgridArray()  ((SubgridArray*)NewSubregionArray())
 
 #define FreeSubgrid(subgrid)  FreeSubregion((Subregion*)subgrid)
 
 #define FreeSubgridArray(subgrid_array) \
-  FreeSubregionArray((SubregionArray*)subgrid_array)
+        FreeSubregionArray((SubregionArray*)subgrid_array)
 
 #define AppendSubgrid(subgrid, subgrid_array) \
-  AppendSubregion((Subregion*)subgrid, (SubregionArray**)subgrid_array)
+        AppendSubregion((Subregion*)subgrid, (SubregionArray**)subgrid_array)
 
-#define AppendSubgridArray(subgrid_array_0, subgrid_array_1) \
-  AppendSubregionArray((SubregionArray*)subgrid_array_0,     \
-                       (SubregionArray**)subgrid_array_1)
+#define AppendSubgridArray(subgrid_array_0, subgrid_array_1)       \
+        AppendSubregionArray((SubregionArray*)subgrid_array_0,     \
+                             (SubregionArray**)subgrid_array_1)
 
 
 /* grid.c */
diff --git a/pftools/head.c b/pftools/head.c
index bfea532fe..382d72ad1 100644
--- a/pftools/head.c
+++ b/pftools/head.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * HHead, PHead
 *
diff --git a/pftools/head.h b/pftools/head.h
index 5bfa261c4..165a82894 100644
--- a/pftools/head.h
+++ b/pftools/head.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Header file for `head.c'
 *
diff --git a/pftools/load.c b/pftools/load.c
index 14ecc911b..623fd817e 100644
--- a/pftools/load.c
+++ b/pftools/load.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <string.h>
 
diff --git a/pftools/load.h b/pftools/load.h
index fb415c540..6eb810f9e 100644
--- a/pftools/load.h
+++ b/pftools/load.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #ifndef LOAD_HEADER
 #define LOAD_HEADER
 
diff --git a/pftools/metaEListBox.tcl b/pftools/metaEListBox.tcl
index 07908ec22..38c62c11e 100644
--- a/pftools/metaEListBox.tcl
+++ b/pftools/metaEListBox.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/metaEntry.tcl b/pftools/metaEntry.tcl
index af3814c0c..7d1e90812 100644
--- a/pftools/metaEntry.tcl
+++ b/pftools/metaEntry.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/parflow.tcl b/pftools/parflow.tcl
index 41bd25826..48da6c388 100644
--- a/pftools/parflow.tcl
+++ b/pftools/parflow.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/pfco.sh b/pftools/pfco.sh
index d9efc43b5..3440d5424 100644
--- a/pftools/pfco.sh
+++ b/pftools/pfco.sh
@@ -1,7 +1,7 @@
 #!/bin/sh
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/pfload_file.h b/pftools/pfload_file.h
index 8a671334c..5fa868446 100644
--- a/pftools/pfload_file.h
+++ b/pftools/pfload_file.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #ifndef _PFLOAD_HEADER
 #define _PFLOAD_HEADER
 
diff --git a/pftools/pflocked.tcl b/pftools/pflocked.tcl
index c81b1d092..818349ff7 100644
--- a/pftools/pflocked.tcl
+++ b/pftools/pflocked.tcl
@@ -4,7 +4,7 @@ exec tclsh "$0" "$@"
 
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/pfmask-to-pfsol.cpp b/pftools/pfmask-to-pfsol.cpp
index 9a8067832..5e4c396ba 100644
--- a/pftools/pfmask-to-pfsol.cpp
+++ b/pftools/pfmask-to-pfsol.cpp
@@ -39,6 +39,7 @@
 #include <cassert>
 #include <cmath>
 #include <limits>
+#include <cfloat>
 
 /*
  * The mask directions.
@@ -356,16 +357,16 @@ int main(int argc, char **argv)
     TCLAP::ValueArg<int> sideArg("","side-patch-label","Side index",false,3,"int");
     cmd.add( sideArg );
 
-    TCLAP::ValueArg<float> zTopArg("","z-top","Set top of domain",false,NAN,"float");
+    TCLAP::ValueArg<float> zTopArg("","z-top","Set top of domain",false,FLT_MIN,"float");
     cmd.add( zTopArg );
 
-    TCLAP::ValueArg<float> zBotArg("","z-bottom","Set bottom of domain",false,NAN,"float");
+    TCLAP::ValueArg<float> zBotArg("","z-bottom","Set bottom of domain",false,FLT_MIN,"float");
     cmd.add( zBotArg );
 
-    TCLAP::ValueArg<float> dxArg("","dx","Set DX",false,NAN,"float");
+    TCLAP::ValueArg<float> dxArg("","dx","Set DX",false,FLT_MIN,"float");
     cmd.add( dxArg );
     
-    TCLAP::ValueArg<float> dyArg("","dy","Set DY",false,NAN,"float");
+    TCLAP::ValueArg<float> dyArg("","dy","Set DY",false,FLT_MIN,"float");
     cmd.add( dyArg );
 
     TCLAP::ValueArg<string>* maskFilenamesArgs[g_maskNames.size()];
@@ -391,7 +392,7 @@ int main(int argc, char **argv)
     }
     else
     {
-      for(int i = 0; i < g_maskNames.size (); ++i)
+      for(int i = 0; i < g_maskNames.size(); ++i)
       {
 	inFilenames[i] = maskFilenamesArgs[i] -> getValue();
       }
@@ -403,7 +404,7 @@ int main(int argc, char **argv)
     side = sideArg.getValue();;
     zTop = zTopArg.getValue();;
     zBot = zBotArg.getValue();;
-    dx = dyArg.getValue();
+    dx = dxArg.getValue();
     dy = dyArg.getValue();
 
   }
@@ -427,7 +428,7 @@ int main(int argc, char **argv)
   } 
   else
   {
-    for(int i = 0; i < inFilenames.size (); ++i)
+    for(int i = 0; i < inFilenames.size(); ++i)
     {
       char* c_filename = strdup(inFilenames[i].c_str());
       databox[i] = loadFile(c_filename);
@@ -441,18 +442,18 @@ int main(int argc, char **argv)
   sx = DataboxX(databox[0]);
   sy = DataboxY(databox[0]);
 
-  if(isnan(dx))
+  if(dx == FLT_MIN)
   {
     dx = DataboxDx(databox[0]);
   }
 
-  if(isnan(dy))
+  if(dy == FLT_MIN)
   {
     dy = DataboxDy(databox[0]);
   }
 
   // If user specifies Top/Bottom on command line override defaults
-  if(isnan(zBot))
+  if(zBot == FLT_MIN)
   {
     sz = 0.0;
   }
@@ -461,7 +462,7 @@ int main(int argc, char **argv)
     sz = zBot;
   }
 
-  if(isnan(zTop))
+  if(zTop == FLT_MIN)
   {
     dz = DataboxDz(databox[0]);
   }
diff --git a/pftools/pfsol-to-vtk.c b/pftools/pfsol-to-vtk.c
index f14e6d374..d5dc26c28 100644
--- a/pftools/pfsol-to-vtk.c
+++ b/pftools/pfsol-to-vtk.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2018, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <stdio.h>
 
diff --git a/pftools/pftappinit.c b/pftools/pftappinit.c
index 1f3db41a2..761e6b051 100644
--- a/pftools/pftappinit.c
+++ b/pftools/pftappinit.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*
  * pftappinit.c
diff --git a/pftools/pftformat.tcl b/pftools/pftformat.tcl
index a4395d0cf..9afc2cc86 100644
--- a/pftools/pftformat.tcl
+++ b/pftools/pftformat.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/pftools.c b/pftools/pftools.c
index 93c54a8e7..6d4d28b46 100644
--- a/pftools/pftools.c
+++ b/pftools/pftools.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /****************************************************************************
  * Program to interactively read, print, etc. different file formats.
  *
@@ -1459,7 +1459,7 @@ int SavePFVTKCommand(
   int i, j, k;
   int flt = 0;
   char          *dzlist_in;
-  char*         Endp1=0;
+  char*         Endp1 = 0;
   int dz_els;
 
   FILE          *fp = NULL;
@@ -1891,15 +1891,15 @@ int SavePFVTKCommand(
  *-----------------------------------------------------------------------*/
 
 int MakePatchySolidCommand(
-                     ClientData  clientData,
-                     Tcl_Interp *interp,
-                     int         argc,
-                     char *      argv[])
+                           ClientData  clientData,
+                           Tcl_Interp *interp,
+                           int         argc,
+                           char *      argv[])
 {
   Data          *data = (Data*)clientData;
 
   char          *filename, *vtk_filename;
-  int           i;
+  int i;
 
   FILE          *fp = NULL;
   FILE          *fp_vtk = NULL;
@@ -1921,7 +1921,7 @@ int MakePatchySolidCommand(
   tophash = NULL;
   bothash = NULL;
 
-  int msk=0, top=0, bot=0, vtk=0, sub_patch=0, bin_out=0;  //Initalize flags for the options
+  int msk = 0, top = 0, bot = 0, vtk = 0, sub_patch = 0, bin_out = 0;  //Initalize flags for the options
   // Note: bin_out is a place holder for a yet to be added BINARY solid file...
   //       vtk is a flag for writing a BINARY VTK of the solid file too
 
@@ -1955,9 +1955,10 @@ int MakePatchySolidCommand(
       vtk = 1;
       vtk_filename = argv[i + 1];
       file1_ext = strrchr(vtk_filename, '.');
-      if (strcmp(file1_ext+1,"vtk")!=0) {
+      if (strcmp(file1_ext + 1, "vtk") != 0)
+      {
         printf("ERROR (pfpatchysolid): vtk file extension must be .vtk\n");
-        printf("                       detected %s\n",file1_ext+1);
+        printf("                       detected %s\n", file1_ext + 1);
         return TCL_ERROR;
       }
     }
@@ -1966,44 +1967,46 @@ int MakePatchySolidCommand(
     {
       filename = argv[i + 1];
       file1_ext = strrchr(filename, '.');
-      if (strcmp(file1_ext+1,"pfsol")!=0) {
+      if (strcmp(file1_ext + 1, "pfsol") != 0)
+      {
         printf("ERROR (pfpatchysolid): ASCII file extension must be .pfsol\n");
-        printf("                       detected %s\n",file1_ext+1);
+        printf("                       detected %s\n", file1_ext + 1);
         return TCL_ERROR;
       }
     }
     else if ((strcmp(argv[i], "-pfsolb") == 0) || (strcmp(argv[i], "–pfsolb") == 0))
     {
-      bin_out=1;
+      bin_out = 1;
       filename = argv[i + 1];
       file1_ext = strrchr(filename, '.');
-      if (strcmp(file1_ext+1,"pfsolb")!=0) {
+      if (strcmp(file1_ext + 1, "pfsolb") != 0)
+      {
         printf("ERROR (pfpatchysolid): Binary file extension must be .pfsolb\n");
-        printf("                       detected %s\n",file1_ext+1);
+        printf("                       detected %s\n", file1_ext + 1);
         return TCL_ERROR;
       }
     }
   }
 
-  if ( (top==0) || (bot==0)) // (msk==0) ||
+  if ((top == 0) || (bot == 0)) // (msk==0) ||
   {
     printf("\n ERROR (pfpatchysolid): Missing required arguments. Please add:\n");
     // if (msk==0)
     // {
     //   printf("      -msk <Mask_dataset_ID>\n");
     // }
-    if (top==0)
+    if (top == 0)
     {
       printf("      -top <Top_surface_dataset_ID>\n");
     }
-    if (top==0)
+    if (top == 0)
     {
       printf("      -bot <Bottom_surface_dataset_ID>\n");
     }
     return TCL_ERROR;
   }
 
-  if (strcmp(filename,"SolidFile.pfsol")==0)
+  if (strcmp(filename, "SolidFile.pfsol") == 0)
   {
     printf("WARNING (pfpatchysolid): No solid file name specified, default is: SolidFile.pfsol\n");
   }
@@ -2020,13 +2023,16 @@ int MakePatchySolidCommand(
     return TCL_ERROR;
   }
 
-  if (msk==1) {
+  if (msk == 1)
+  {
     if ((databox = DataMember(data, maskkey, entryPtr)) == NULL)
     {
       SetNonExistantError(interp, maskkey);
       return TCL_ERROR;
     }
-  } else {
+  }
+  else
+  {
     // No mask provided so make one of all ones matching the size of top
     int NX = DataboxNx(top_databox);
     int NY = DataboxNy(top_databox);
@@ -2038,7 +2044,7 @@ int MakePatchySolidCommand(
     double DY = DataboxDy(top_databox);
     double DZ = DataboxDz(top_databox);
 
-    databox=NewDataboxDefault(NX,NY,NZ,X,Y,Z,DX,DY,DZ,1.0);
+    databox = NewDataboxDefault(NX, NY, NZ, X, Y, Z, DX, DY, DZ, 1.0);
   }
 
   if ((fp = fopen(filename, "wb")) == NULL)
@@ -2047,7 +2053,7 @@ int MakePatchySolidCommand(
     ReadWriteError(interp);
     return TCL_ERROR;
   }
-  if (vtk==1)
+  if (vtk == 1)
   {
     if ((fp_vtk = fopen(vtk_filename, "wb")) == NULL)
     {
@@ -2062,21 +2068,21 @@ int MakePatchySolidCommand(
   // start_time=clock();
   // double run_time;
 
-  i=MakePatchySolid(fp, fp_vtk, databox, top_databox, bot_databox, sub_patch, bin_out);
+  i = MakePatchySolid(fp, fp_vtk, databox, top_databox, bot_databox, sub_patch, bin_out);
 
   // end_time=clock();
   // run_time=(double)(end_time-start_time) / CLOCKS_PER_SEC;
   // printf("Elapsed time: %f\n", run_time);
 
-  if (i!=0)
+  if (i != 0)
   {
-    if (i==-2) // Flag for errors deriving from input issues
+    if (i == -2) // Flag for errors deriving from input issues
     {
       printf("\n ERROR (pfpatchysolid): Error with inputs\n");
     }
     else  // Everything else...
     {
-    printf("\n ERROR (pfpatchysolid): Other internal error\n");
+      printf("\n ERROR (pfpatchysolid): Other internal error\n");
     }
   }
 
@@ -2100,60 +2106,65 @@ int MakePatchySolidCommand(
 //   from first filename format to the second filename format
 // ----------------------------------------------------------------------------
 int pfsolFmtConvert(
-                     ClientData  clientData,
-                     Tcl_Interp *interp,
-                     int         argc,
-                     char *      argv[])
+                    ClientData  clientData,
+                    Tcl_Interp *interp,
+                    int         argc,
+                    char *      argv[])
 {
   char          *bin_filename, *ascii_filename;
-  char          *file1_name, *file2_name,*file1_ext, *file2_ext;
-  int           bin2asc;
+  char          *file1_name, *file2_name, *file1_ext, *file2_ext;
+  int bin2asc;
 
   FILE          *fp_bin = NULL;
   FILE          *fp_ascii = NULL;
 
   // Perform some checks before calling the appropriate routine in solidtools.c
 
-  if (argc!=3)
+  if (argc != 3)
   {
     printf("ERROR (pfsolidfmtconvert): Two input file names are required\n");
     return TCL_ERROR;
   }
 
-  file1_name=argv[1];
-  file2_name=argv[2];
+  file1_name = argv[1];
+  file2_name = argv[2];
 
   file1_ext = strrchr(file1_name, '.');
   file2_ext = strrchr(file2_name, '.');
 
-  if (!file1_ext) {
+  if (!file1_ext)
+  {
     printf("ERROR (pfsolidfmtconvert): Missing extension on file 1, must be .pfsol or .pfsolb\n");
     return TCL_ERROR;
   }
-  if (!file2_ext) {
+  if (!file2_ext)
+  {
     printf("ERROR (pfsolidfmtconvert): Missing extension on file 2, must be .pfsol or .pfsolb\n");
     return TCL_ERROR;
   }
-  if ((strcmp(file1_ext+1,"pfsol")!=0)&(strcmp(file1_ext+1,"pfsolb")!=0)) {
+  if ((strcmp(file1_ext + 1, "pfsol") != 0) && (strcmp(file1_ext + 1, "pfsolb") != 0))
+  {
     printf("ERROR (pfsolidfmtconvert): File extension on file 1 must be .pfsol or .pfsolb\n");
-    printf("                           detected .%s\n",file1_ext+1);
+    printf("                           detected .%s\n", file1_ext + 1);
     return TCL_ERROR;
   }
-  if ((strcmp(file2_ext+1,"pfsol")!=0)&(strcmp(file2_ext+1,"pfsolb")!=0)) {
+  if ((strcmp(file2_ext + 1, "pfsol") != 0) && (strcmp(file2_ext + 1, "pfsolb") != 0))
+  {
     printf("ERROR (pfsolidfmtconvert): File extension on file 2 must be .pfsol or .pfsolb\n");
-    printf("                           detected .%s\n",file2_ext+1);
+    printf("                           detected .%s\n", file2_ext + 1);
     return TCL_ERROR;
   }
-  if (strcmp(file2_ext+1,file1_ext+1)==0) {
+  if (strcmp(file2_ext + 1, file1_ext + 1) == 0)
+  {
     printf("ERROR (pfsolidfmtconvert): File extensions must be different\n");
     return TCL_ERROR;
   }
 
-  if (strcmp(file1_ext+1,"pfsol")==0)
+  if (strcmp(file1_ext + 1, "pfsol") == 0)
   {
-    ascii_filename=file1_name;
-    bin_filename=file2_name;
-    bin2asc=0;
+    ascii_filename = file1_name;
+    bin_filename = file2_name;
+    bin2asc = 0;
 
     if ((fp_ascii = fopen(ascii_filename, "r")) == NULL)
     {
@@ -2167,10 +2178,12 @@ int pfsolFmtConvert(
       ReadWriteError(interp);
       return TCL_ERROR;
     }
-  } else {
-    ascii_filename=file2_name;
-    bin_filename=file1_name;
-    bin2asc=1;
+  }
+  else
+  {
+    ascii_filename = file2_name;
+    bin_filename = file1_name;
+    bin2asc = 1;
 
     if ((fp_ascii = fopen(ascii_filename, "w")) == NULL)
     {
@@ -2186,11 +2199,14 @@ int pfsolFmtConvert(
     }
   }
 
-  int out_status=0;
-  if (bin2asc==0){
-    out_status=ConvertPfsolAscii2Bin(fp_ascii,fp_bin);
-  } else {
-    out_status=ConvertPfsolBin2Ascii(fp_bin,fp_ascii);
+  int out_status = 0;
+  if (bin2asc == 0)
+  {
+    out_status = ConvertPfsolAscii2Bin(fp_ascii, fp_bin);
+  }
+  else
+  {
+    out_status = ConvertPfsolBin2Ascii(fp_bin, fp_ascii);
   }
 
   if (fp_ascii)
@@ -2202,7 +2218,7 @@ int pfsolFmtConvert(
     fclose(fp_bin);
   }
 
-  if (out_status!=0)
+  if (out_status != 0)
   {
     printf("\n ERROR (pfsolidfmtconvert): Problem during file conversion\n");
     return TCL_ERROR;
diff --git a/pftools/pftools.h b/pftools/pftools.h
index 899e520a0..3f180f2f9 100644
--- a/pftools/pftools.h
+++ b/pftools/pftools.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Header file for `pftools' program
 *
@@ -80,9 +80,9 @@ typedef struct {
 #define DataGridType(data)  ((data)->grid_type)
 #define DataTotalMem(data)  ((data)->total_members)
 #define DataNum(data)       ((data)->num)
-#define DataMember(data, hashkey, entryPtr)                           \
-  (((entryPtr = Tcl_FindHashEntry(&DataMembers(data), hashkey)) != 0) \
-   ? (Databox*)Tcl_GetHashValue(entryPtr)                             \
+#define DataMember(data, hashkey, entryPtr)                                 \
+        (((entryPtr = Tcl_FindHashEntry(&DataMembers(data), hashkey)) != 0) \
+   ? (Databox*)Tcl_GetHashValue(entryPtr)                                   \
    : (Databox*)NULL)
 #define FreeData(data) (free((Data*)data))
 
@@ -151,9 +151,9 @@ int GWStorageCommand(ClientData clientData, Tcl_Interp *interp, int argc, char *
 int SurfaceRunoffCommand(ClientData clientData, Tcl_Interp *interp, int argc, char *argv []);
 int WaterTableDepthCommand(ClientData clientData, Tcl_Interp *interp, int argc, char *argv []);
 
-int SavePFVTKCommand (ClientData clientData, Tcl_Interp *interp, int argc, char *argv []);
-int MakePatchySolidCommand (ClientData clientData, Tcl_Interp *interp, int argc, char *argv []);
-int pfsolFmtConvert (ClientData clientData, Tcl_Interp *interp, int argc, char *argv []);
+int SavePFVTKCommand(ClientData clientData, Tcl_Interp *interp, int argc, char *argv []);
+int MakePatchySolidCommand(ClientData clientData, Tcl_Interp *interp, int argc, char *argv []);
+int pfsolFmtConvert(ClientData clientData, Tcl_Interp *interp, int argc, char *argv []);
 
 void Axpy(double alpha, Databox *X, Databox *Y);
 void Sum(Databox *X, double *sum);
diff --git a/pftools/pfvtk.tcl b/pftools/pfvtk.tcl
index 0d300282b..237a1347c 100644
--- a/pftools/pfvtk.tcl
+++ b/pftools/pfvtk.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/pfwell_cat.c b/pftools/pfwell_cat.c
index 3c29ea1cc..1fc16ad88 100644
--- a/pftools/pfwell_cat.c
+++ b/pftools/pfwell_cat.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "pfwell_cat.h"
 
diff --git a/pftools/pfwell_cat.h b/pftools/pfwell_cat.h
index 7a0b5f955..93010018a 100644
--- a/pftools/pfwell_cat.h
+++ b/pftools/pfwell_cat.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 * Header file for `well.c'
diff --git a/pftools/pfwell_data.c b/pftools/pfwell_data.c
index 2c8fa6921..fd120d378 100644
--- a/pftools/pfwell_data.c
+++ b/pftools/pfwell_data.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "pfwell_data.h"
 
@@ -34,8 +34,8 @@ int main(
 {
   WellListMember   *head;
   WellListMember   *member,
-    *new_member,
-    *old_member = NULL;
+                   *new_member,
+                   *old_member = NULL;
 
   Background       *background;
   ProblemData      *problem_data;
diff --git a/pftools/pfwell_data.h b/pftools/pfwell_data.h
index 325502f68..a4793d680 100644
--- a/pftools/pfwell_data.h
+++ b/pftools/pfwell_data.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 * Header file for `well.c'
@@ -73,25 +73,25 @@ typedef struct well_list_member {
  * Accessor macros: WellListMember
  *--------------------------------------------------------------------------*/
 #define WellListMemberTime(well_list_member) \
-  ((well_list_member)->time)
+        ((well_list_member)->time)
 
 #define WellListMemberWellDataPhysicals(well_list_member) \
-  ((well_list_member)->well_data_physicals)
+        ((well_list_member)->well_data_physicals)
 #define WellListMemberWellDataPhysical(well_list_member, i) \
-  ((well_list_member)->well_data_physicals[i])
+        ((well_list_member)->well_data_physicals[i])
 
 #define WellListMemberWellDataValues(well_list_member) \
-  ((well_list_member)->well_data_values)
+        ((well_list_member)->well_data_values)
 #define WellListMemberWellDataValue(well_list_member, i) \
-  ((well_list_member)->well_data_values[i])
+        ((well_list_member)->well_data_values[i])
 
 #define WellListMemberWellDataStats(well_list_member) \
-  ((well_list_member)->well_data_stats)
+        ((well_list_member)->well_data_stats)
 #define WellListMemberWellDataStat(well_list_member, i) \
-  ((well_list_member)->well_data_stats[i])
+        ((well_list_member)->well_data_stats[i])
 
 #define WellListMemberNextWellListMember(well_list_member) \
-  ((well_list_member)->next_well_list_member)
+        ((well_list_member)->next_well_list_member)
 
 /*-----------------------------------------------------------------------
  * function prototypes
diff --git a/pftools/pfwrite.tcl b/pftools/pfwrite.tcl
index e549ab8a9..bc1d74522 100644
--- a/pftools/pfwrite.tcl
+++ b/pftools/pfwrite.tcl
@@ -4,7 +4,7 @@ exec tclsh "$0" "$@"
 
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/prepostproc/BOV_vis_file.test.tcl b/pftools/prepostproc/BOV_vis_file.test.tcl
old mode 100755
new mode 100644
diff --git a/pftools/prepostproc/PFB-ReadFcn.R b/pftools/prepostproc/PFB-ReadFcn.R
index 905ba859d..6cdd0908c 100644
--- a/pftools/prepostproc/PFB-ReadFcn.R
+++ b/pftools/prepostproc/PFB-ReadFcn.R
@@ -1,20 +1,20 @@
-readpfb=function(filename, verbose){
-
+readpfb=function(filename, verbose){
+
 # code to read parflow binary in R
 # RMM 10-19-13
 
-to.read = file(filename,"rb")
-on.exit(close(to.read))
+to.read = file(filename,"rb")
+on.exit(close(to.read))
 if(verbose){print(to.read)}
 #read in X0, Y0, Z0 of domain
 X = readBin(to.read,double(),endian="big")
 Y = readBin(to.read,double(),endian="big")
 Z = readBin(to.read,double(),endian="big")
-
+
 if(verbose){
 print(X)
 print(Y)
-print(Z)
+print(Z)
 }
 
 #read in global nx, ny, nz of domain
@@ -24,18 +24,18 @@ nz = readBin(to.read,integer(),endian="big")
 
 # set up a blank array the size of the domain
 Data = array(0,dim=c(nx,ny,nz))
-
+
 if(verbose){
 print(nx)
 print(ny)
-print(nz)
+print(nz)
 }
 
 #read in dx dy dz
 dx = readBin(to.read,double(),endian="big")
 dy = readBin(to.read,double(),endian="big")
 dz = readBin(to.read,double(),endian="big")
-
+
 if(verbose){
 print(dx)
 print(dy)
@@ -52,33 +52,33 @@ for (i in 1:is ) {
 ix = readBin(to.read,integer(),endian="big")
 iy = readBin(to.read,integer(),endian="big")
 iz = readBin(to.read,integer(),endian="big")
-
-if(verbose){
+
+if(verbose){
 print(ix)
 print(iy)
-print(iz)
+print(iz)
 }
 
 #read in locall nx, ny, nz of this subgrid
 nnx = readBin(to.read,integer(),endian="big")
 nny = readBin(to.read,integer(),endian="big")
 nnz = readBin(to.read,integer(),endian="big")
-
-if(verbose){
+
+if(verbose){
 print(nnx)
 print(nny)
 print(nnz)
-}
+}
 
 #read in local refinement of this subgridr
 rx = readBin(to.read,integer(),endian="big")
 ry = readBin(to.read,integer(),endian="big")
 rz = readBin(to.read,integer(),endian="big")
-
-if(verbose){
+
+if(verbose){
 print(rx)
 print(ry)
-print(rz)
+print(rz)
 }
 
 for (k in (iz+1):(iz+nnz))  {
@@ -89,5 +89,5 @@ for (k in (iz+1):(iz+nnz))  {
    }
   }
 }
-return(Data)
+return(Data)
 }
\ No newline at end of file
diff --git a/pftools/prepostproc/pfb2nc.py b/pftools/prepostproc/pfb2nc.py
index 7721a7f1d..62a0c062e 100644
--- a/pftools/prepostproc/pfb2nc.py
+++ b/pftools/prepostproc/pfb2nc.py
@@ -10,7 +10,7 @@
 ## Added another time dimension and loop over a list of files for time variable.
 ####################################################################
 
-#Requires numPy and netcdf python bindings.
+# Requires numPy and netcdf python bindings.
 import sys
 from struct import *
 import numpy as np
@@ -19,123 +19,124 @@
 
 # Read command line input
 fileName = sys.argv[1]
-outFile  = sys.argv[2]
-varName  = sys.argv[3]
+outFile = sys.argv[2]
+varName = sys.argv[3]
 
 fileList = open(fileName, "r")
 lines = fileList.readlines()
-fileName = [ii.rstrip('\n') for ii in lines]
+fileName = [ii.rstrip("\n") for ii in lines]
 
-#Set the time axis. Change the time incremente suitable to your experiment.
-timeArray = [datetime(1990,2,1) + ii*timedelta(hours=3) for ii in range(0,len(fileName))]
+# Set the time axis. Change the time incremente suitable to your experiment.
+timeArray = [
+    datetime(1990, 2, 1) + ii * timedelta(hours=3) for ii in range(0, len(fileName))
+]
 
 
 # Define netCDF dimensions (Adapt for your domain dimensions)
-#Following dimensions correspond to Little Washita test case
-ncfile = Dataset(outFile,'w',format="NETCDF4")
-ncfile.createDimension('time',None)
-ncfile.createDimension('lon',41)
-ncfile.createDimension('lat',41)
-ncfile.createDimension('lev',50)
+# Following dimensions correspond to Little Washita test case
+ncfile = Dataset(outFile, "w", format="NETCDF4")
+ncfile.createDimension("time", None)
+ncfile.createDimension("lon", 41)
+ncfile.createDimension("lat", 41)
+ncfile.createDimension("lev", 50)
 
 # Create a variable
-pfVar = ncfile.createVariable(varName,'f8',('time', 'lev','lat','lon'))
-#Create time variable and add data to it
-timeVar = ncfile.createVariable("time",'f8',('time'))
+pfVar = ncfile.createVariable(varName, "f8", ("time", "lev", "lat", "lon"))
+# Create time variable and add data to it
+timeVar = ncfile.createVariable("time", "f8", ("time"))
 timeVar.units = "hours since 1990-02-01 00:00:00.0"
 timeVar.calendar = "gregorian"
-timeVar[:] = date2num(timeArray,units=timeVar.units,calendar=timeVar.calendar)
+timeVar[:] = date2num(timeArray, units=timeVar.units, calendar=timeVar.calendar)
 
 # Allocate an array to read the data from pfb(Adapt for your domain dimensions)
-#Following dimensions correspond to Little Washita test case
-press = np.empty([50,41,41])
-
-for fname in range(0,len(fileName)):
-	file = open(fileName[fname], 'rb')
-	print(fileName[fname])
-	# Read the start index of the domain
-	bigEnd = file.read(8)
-	x1 = list(unpack('>d', bigEnd))[0]
-	#print(x1)
-	bigEnd = file.read(8)
-	y1 = list(unpack('>d', bigEnd))[0]
-	#print(y1)
-	bigEnd = file.read(8)
-	z1 = list(unpack('>d', bigEnd))[0]
-	#print(z1)
-
-
-	# Read the number of points in x, y and z direction
-	bigEnd = file.read(4)
-	nx = list(unpack('>i', bigEnd))[0]
-	#print(nx)
-	bigEnd = file.read(4)
-	ny = list(unpack('>i', bigEnd))[0]
-	#print(ny)
-	bigEnd = file.read(4)
-	nz = list(unpack('>i', bigEnd))[0]
-	#print(nz)
-
-
-	# Read dx, dy and dz
-	bigEnd = file.read(8)
-	dx = list(unpack('>d', bigEnd))[0]
-	#print(dx)
-	bigEnd = file.read(8)
-	dy = list(unpack('>d', bigEnd))[0]
-	#print(dy)
-	bigEnd = file.read(8)
-	dz = list(unpack('>d', bigEnd))[0]
-	#print(dz)
-	
-	# Allocate array to read the data
-	
-	
-	# Read the number of subdomains =  number of procs
-	bigEnd = file.read(4)
-	nSubGrid = list(unpack('>i', bigEnd))[0]
-	#print(nSubGrid)
-	
-	for gridCounter in range(0, nSubGrid):
-	
-	# Read the subgrid indices and counters
-		bigEnd = file.read(4)
-		ix = list(unpack('>i', bigEnd))[0]
-		#print(ix)
-		bigEnd = file.read(4)
-		iy = list(unpack('>i', bigEnd))[0]
-		#print(iy)
-		bigEnd = file.read(4)
-		iz = list(unpack('>i', bigEnd))[0]
-		#print(iz)
-	
-		bigEnd = file.read(4)
-		nnx = list(unpack('>i', bigEnd))[0]
-		#print(nnx)
-		bigEnd = file.read(4)
-		nny = list(unpack('>i', bigEnd))[0]
-		#print(nny)
-		bigEnd = file.read(4)
-		nnz = list(unpack('>i', bigEnd))[0]
-		#print(nnz)
-		
-		bigEnd = file.read(4)
-		rx = list(unpack('>i', bigEnd))[0]
-		#print(rx)
-		bigEnd = file.read(4)
-		ry = list(unpack('>i', bigEnd))[0]
-		#print(ry)
-		bigEnd = file.read(4)
-		rz = list(unpack('>i', bigEnd))[0]
-		#print(rz)
-
-		bigEnd = file.read(nnx*nny*nnz*8)
-		fmt = ">%dd" % (nnx*nny*nnz)
-		temp = list(unpack(fmt,bigEnd))
-		press[iz:iz+nnz,iy:iy+nny,ix:ix+nnx] = np.reshape(temp,(nnz,nny,nnx))
-	
-	# Write into netCDF variable
-	pfVar[fname,:,:,:] = press
+# Following dimensions correspond to Little Washita test case
+press = np.empty([50, 41, 41])
+
+for fname in range(0, len(fileName)):
+    file = open(fileName[fname], "rb")
+    print(fileName[fname])
+    # Read the start index of the domain
+    bigEnd = file.read(8)
+    x1 = list(unpack(">d", bigEnd))[0]
+    # print(x1)
+    bigEnd = file.read(8)
+    y1 = list(unpack(">d", bigEnd))[0]
+    # print(y1)
+    bigEnd = file.read(8)
+    z1 = list(unpack(">d", bigEnd))[0]
+    # print(z1)
+
+    # Read the number of points in x, y and z direction
+    bigEnd = file.read(4)
+    nx = list(unpack(">i", bigEnd))[0]
+    # print(nx)
+    bigEnd = file.read(4)
+    ny = list(unpack(">i", bigEnd))[0]
+    # print(ny)
+    bigEnd = file.read(4)
+    nz = list(unpack(">i", bigEnd))[0]
+    # print(nz)
+
+    # Read dx, dy and dz
+    bigEnd = file.read(8)
+    dx = list(unpack(">d", bigEnd))[0]
+    # print(dx)
+    bigEnd = file.read(8)
+    dy = list(unpack(">d", bigEnd))[0]
+    # print(dy)
+    bigEnd = file.read(8)
+    dz = list(unpack(">d", bigEnd))[0]
+    # print(dz)
+
+    # Allocate array to read the data
+
+    # Read the number of subdomains =  number of procs
+    bigEnd = file.read(4)
+    nSubGrid = list(unpack(">i", bigEnd))[0]
+    # print(nSubGrid)
+
+    for gridCounter in range(0, nSubGrid):
+
+        # Read the subgrid indices and counters
+        bigEnd = file.read(4)
+        ix = list(unpack(">i", bigEnd))[0]
+        # print(ix)
+        bigEnd = file.read(4)
+        iy = list(unpack(">i", bigEnd))[0]
+        # print(iy)
+        bigEnd = file.read(4)
+        iz = list(unpack(">i", bigEnd))[0]
+        # print(iz)
+
+        bigEnd = file.read(4)
+        nnx = list(unpack(">i", bigEnd))[0]
+        # print(nnx)
+        bigEnd = file.read(4)
+        nny = list(unpack(">i", bigEnd))[0]
+        # print(nny)
+        bigEnd = file.read(4)
+        nnz = list(unpack(">i", bigEnd))[0]
+        # print(nnz)
+
+        bigEnd = file.read(4)
+        rx = list(unpack(">i", bigEnd))[0]
+        # print(rx)
+        bigEnd = file.read(4)
+        ry = list(unpack(">i", bigEnd))[0]
+        # print(ry)
+        bigEnd = file.read(4)
+        rz = list(unpack(">i", bigEnd))[0]
+        # print(rz)
+
+        bigEnd = file.read(nnx * nny * nnz * 8)
+        fmt = ">%dd" % (nnx * nny * nnz)
+        temp = list(unpack(fmt, bigEnd))
+        press[iz : iz + nnz, iy : iy + nny, ix : ix + nnx] = np.reshape(
+            temp, (nnz, nny, nnx)
+        )
+
+    # Write into netCDF variable
+    pfVar[fname, :, :, :] = press
 
 
 ncfile.close()
diff --git a/pftools/printdatabox.c b/pftools/printdatabox.c
index c67128f38..9e7548637 100644
--- a/pftools/printdatabox.c
+++ b/pftools/printdatabox.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Print routines for pftools
 *
@@ -231,7 +231,7 @@ void            PrintAVSField(
  * print a Databox in HDF SDS format
  *-----------------------------------------------------------------------*/
 
-int             PrintSDS(filename, type, v)
+int PrintSDS(filename, type, v)
 char            *filename;
 int type;
 Databox         *v;
@@ -726,7 +726,7 @@ void            PrintVTK(
     double *DTd;
     float *DTf;
     // DTf = (float*)malloc(tools_SizeofFloat * nxyzp);
-    DTf = (float*)calloc(nxyzp,tools_SizeofFloat);
+    DTf = (float*)calloc(nxyzp, tools_SizeofFloat);
     DTd = DataboxCoeffs(v);
     for (j = 0; j < (NX * NY * NZ); ++j)
     {
@@ -776,7 +776,7 @@ void            PrintTFG_VTK(
   int i;
   float *pnt;
   // pnt = (float*)malloc(tools_SizeofFloat * nxyzp * 3);
-  pnt = (float*)calloc(nxyzp * 3,tools_SizeofFloat);
+  pnt = (float*)calloc(nxyzp * 3, tools_SizeofFloat);
   for (i = 0; i < (nxyzp * 3); ++i)
   {
     pnt[i] = (float)pnts[i];
@@ -801,7 +801,7 @@ void            PrintTFG_VTK(
     double *DTd;
     float *DTf;
     // DTf = (float*)malloc(tools_SizeofFloat * nxyzp);
-    DTf = (float*)calloc(nxyzp,tools_SizeofFloat);
+    DTf = (float*)calloc(nxyzp, tools_SizeofFloat);
     DTd = DataboxCoeffs(v);
 
     for (j = 0; j < (NX * NY * NZ); ++j)
@@ -881,7 +881,7 @@ void            PrintCLMVTK(
     float  *DTf;
     float  *val;
     // DTf = (float*)malloc(tools_SizeofFloat * nxyzp);
-    DTf = (float*)calloc(nxyzp,tools_SizeofFloat);
+    DTf = (float*)calloc(nxyzp, tools_SizeofFloat);
     DTd = DataboxCoeffs(v);
     for (j = 0; j < (NX * NY * NZ); ++j)
     {
@@ -1001,7 +1001,7 @@ void            PrintTFG_CLMVTK(
 //    int     i;
   float *pnt;
   // pnt = (float*)malloc(tools_SizeofFloat * nxyzp * 3);
-  pnt = (float*)calloc(nxyzp * 3,tools_SizeofFloat);
+  pnt = (float*)calloc(nxyzp * 3, tools_SizeofFloat);
   for (i = 0; i < (nxyzp * 3); ++i)
   {
     pnt[i] = (float)pnts[i];
@@ -1029,7 +1029,7 @@ void            PrintTFG_CLMVTK(
     float  *DTf;
     float  *val;
     // DTf = (float*)malloc(tools_SizeofFloat * nxyzp);
-    DTf = (float*)calloc(nxyzp,tools_SizeofFloat);
+    DTf = (float*)calloc(nxyzp, tools_SizeofFloat);
     DTd = DataboxCoeffs(v);
     for (j = 0; j < (NX * NY * NZ); ++j)
     {
diff --git a/pftools/printdatabox.h b/pftools/printdatabox.h
index 97cb6b56f..d5449f9c4 100644
--- a/pftools/printdatabox.h
+++ b/pftools/printdatabox.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Header file for `printdatabox.c'
 *
diff --git a/pftools/projecttin.c b/pftools/projecttin.c
index 136d0d4fa..4b1dba62a 100644
--- a/pftools/projecttin.c
+++ b/pftools/projecttin.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <stdio.h>
 #include <math.h>
@@ -473,7 +473,7 @@ void AddTINToProjectionMesh(ProjectionMesh *PM, gms_TIN *tin)
       {
         for (i = 0; i < projection_point->num_triangles; i++)
         {
-	  //MeshTriangle *projection_triangle;
+          //MeshTriangle *projection_triangle;
           //projection_triangle = &(PM->triangles[projection_point->triangles[i]]);
           /* Add Intersection point to the intersection list */
           AddIntersectionPointToMeshVertex(projection_point,
@@ -848,7 +848,7 @@ void IntersectLineWithTriangle(
     q[1] = NAN;
     p[2] = NAN;
     q[2] = NAN;
-  }    
+  }
 
   /*---------------------------------------------
    * Check to see how many edges cross the
diff --git a/pftools/python/CMakeLists.txt b/pftools/python/CMakeLists.txt
index 574ab389a..654b7b593 100644
--- a/pftools/python/CMakeLists.txt
+++ b/pftools/python/CMakeLists.txt
@@ -11,36 +11,56 @@ add_custom_command(
     "${CMAKE_CURRENT_SOURCE_DIR}"
     "${CMAKE_CURRENT_BINARY_DIR}"
   DEPENDS
-    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/__init__.py"
+    "${CMAKE_CURRENT_SOURCE_DIR}/CHANGELOG.md"
+    "${CMAKE_CURRENT_SOURCE_DIR}/MANIFEST.in"
+    "${CMAKE_CURRENT_SOURCE_DIR}/README.md"
     "${CMAKE_CURRENT_SOURCE_DIR}/parflow/cli/__init__.py"
     "${CMAKE_CURRENT_SOURCE_DIR}/parflow/cli/pfdist_sort.py"
     "${CMAKE_CURRENT_SOURCE_DIR}/parflow/cli/tcl2py.py"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/database/__init__.py"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/database/core.py"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/database/domains.py"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/database/handlers.py"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/ref/default_vegp_values.json"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/ref/reservoir_keys.yaml"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/ref/subsurface_conus_1.txt"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/ref/subsurface_freeze_cherry.txt"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/ref/subsurface_washita.txt"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/ref/table_keys.yaml"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/ref/vegp_header.txt"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/ref/vegp_igbp.txt"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/ref/vegp_keys.yaml"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/tests/__init__.py"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/tests/data/__init__.py"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/tests/data/forsyth5.out.press.00000.pfb"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/tests/data/forsyth5.out.press.00001.pfb"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/tests/pfb_summary.py"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/tests/test_pf_xarray.py"
     "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/__init__.py"
     "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/builders.py"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/compare.py"
     "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/core.py"
     "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/export.py"
     "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/fs.py"
     "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/helper.py"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/hydrology.py"
     "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/io.py"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/pf_backend.py"
     "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/settings.py"
     "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/terminal.py"
-    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/database/__init__.py"
-    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/database/core.py"
-    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/database/domains.py"
-    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/database/handlers.py"
-    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/ref/table_keys.yaml"
-    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/ref/vegp_keys.yaml"
-    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/ref/subsurface_conus_1.txt"
-    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/ref/subsurface_freeze_cherry.txt"
-    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/ref/subsurface_washita.txt"
-    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/ref/vegp_igbp.txt"
-    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/ref/vegp_header.txt"
-    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/ref/default_vegp_values.json"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/top.py"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/tools/util.py"
+    "${CMAKE_CURRENT_SOURCE_DIR}/parflow/__init__.py"
+    "${CMAKE_CURRENT_SOURCE_DIR}/requirements.txt"
+    "${CMAKE_CURRENT_SOURCE_DIR}/requirements_all.txt"
+    "${CMAKE_CURRENT_SOURCE_DIR}/requirements_pfsol.txt"
+    "${CMAKE_CURRENT_SOURCE_DIR}/setup.cfg"
+    "${CMAKE_CURRENT_SOURCE_DIR}/requirements_dev.txt"
     "${CMAKE_CURRENT_SOURCE_DIR}/setup.py"
-    "${CMAKE_CURRENT_SOURCE_DIR}/README.md"
-    "${CMAKE_CURRENT_SOURCE_DIR}/MANIFEST.in"
+    "${CMAKE_CURRENT_SOURCE_DIR}/pfpython"
+    "${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt"
   COMMENT
-    "Copying Python pftools to the build directory"
+  "Copying Python pftools to the build directory"
 )
 
 # Install Python package
@@ -62,13 +82,12 @@ install(
 )
 install(
   PROGRAMS
-    "${CMAKE_CURRENT_SOURCE_DIR}/py-setup"
     "${CMAKE_CURRENT_SOURCE_DIR}/pfpython"
   DESTINATION
     "bin"
-)
+    )
 
-install(SCRIPT "${CMAKE_SOURCE_DIR}/pftools/python/PostInstall.cmake")
+install(CODE "execute_process(COMMAND ${CMAKE_INSTALL_PREFIX}/py-env/bin/pip3 install ${CMAKE_INSTALL_PREFIX}/python)" )
 
 add_custom_target(
   pf-python ALL
@@ -89,7 +108,7 @@ add_custom_target(PythonCreatePackage
   COMMAND
       ${PARFLOW_PYTHON} setup.py sdist bdist_wheel
   COMMAND
-      twine check "${CMAKE_CURRENT_BINARY_DIR}/dist/*"
+      ${CMAKE_INSTALL_PREFIX}/py-env/bin/twine check "${CMAKE_CURRENT_BINARY_DIR}/dist/*"
   COMMENT
       "Updating the pftools Python package"
   VERBATIM
@@ -101,7 +120,7 @@ add_custom_target(PythonPublishPackage
   WORKING_DIRECTORY
       "${CMAKE_CURRENT_BINARY_DIR}"
   COMMAND
-      twine upload dist/*
+      ${CMAKE_INSTALL_PREFIX}/py-env/bin/twine upload dist/*
   COMMENT
       "Publishing the pftools Python package"
   VERBATIM
diff --git a/pftools/python/PostInstall.cmake b/pftools/python/PostInstall.cmake
deleted file mode 100644
index adf97f496..000000000
--- a/pftools/python/PostInstall.cmake
+++ /dev/null
@@ -1,3 +0,0 @@
-execute_process(
-  COMMAND "${CMAKE_INSTALL_PREFIX}/bin/py-setup"
-)
diff --git a/pftools/python/parflow/__init__.py b/pftools/python/parflow/__init__.py
index 9544e7bef..7432b17f0 100644
--- a/pftools/python/parflow/__init__.py
+++ b/pftools/python/parflow/__init__.py
@@ -6,11 +6,14 @@
 """
 from .tools import Run
 from .tools import ParflowBinaryReader, read_pfb, read_pfb_sequence, write_pfb
+from .tools import pf_test_file, pf_test_file_with_abs
 
 __all__ = [
-    'Run',
-    'ParflowBinaryReader',
-    'read_pfb',
-    'write_pfb',
-    'read_pfb_sequence',
+    "Run",
+    "ParflowBinaryReader",
+    "read_pfb",
+    "write_pfb",
+    "read_pfb_sequence",
+    "pf_test_file",
+    "pf_test_file_with_abs",
 ]
diff --git a/pftools/python/parflow/cli/pfdist_sort.py b/pftools/python/parflow/cli/pfdist_sort.py
index 3c6745744..4b374e925 100644
--- a/pftools/python/parflow/cli/pfdist_sort.py
+++ b/pftools/python/parflow/cli/pfdist_sort.py
@@ -8,7 +8,7 @@
 
 def write_sorted_pfidb(input_file, output_file=None):
     if not os.path.exists(input_file):
-        print(f'Input file does not exist: {input_file}')
+        print(f"Input file does not exist: {input_file}")
         return
 
     if output_file is None:
@@ -19,14 +19,20 @@ def write_sorted_pfidb(input_file, output_file=None):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Parflow PFidb sorter")
-    parser.add_argument("--input-file", "-i",
-                        default=None,
-                        dest="input",
-                        help="Path to ParFlow database file to sort")
-    parser.add_argument("--output-file", "-o",
-                        default=None,
-                        dest="output",
-                        help="Output file path to write sorted result to")
+    parser.add_argument(
+        "--input-file",
+        "-i",
+        default=None,
+        dest="input",
+        help="Path to ParFlow database file to sort",
+    )
+    parser.add_argument(
+        "--output-file",
+        "-o",
+        default=None,
+        dest="output",
+        help="Output file path to write sorted result to",
+    )
     args = parser.parse_args()
 
     if args.input is None:
diff --git a/pftools/python/parflow/cli/tcl2py.py b/pftools/python/parflow/cli/tcl2py.py
index 99aa8d3b0..e73cb2373 100644
--- a/pftools/python/parflow/cli/tcl2py.py
+++ b/pftools/python/parflow/cli/tcl2py.py
@@ -9,89 +9,103 @@ def tcl_to_python(input_file, output_file=None, run_name=None):
             run_name = Path(input_file).stem
 
         if output_file is None:
-            output_file = str(Path(input_file).with_suffix('.py'))
+            output_file = str(Path(input_file).with_suffix(".py"))
     except Exception:
-        print(f'Invalid input file: {input_file}')
+        print(f"Invalid input file: {input_file}")
         return
 
     if not Path(input_file).exists():
-        print(f'Input file does not exist: {input_file}')
+        print(f"Input file does not exist: {input_file}")
         return
 
-    run_str = str(run_name) + '.'
-    with open(input_file, 'r') as fin:
-        with open(output_file, 'w') as fout:
-            prev_line = ''
+    run_str = str(run_name) + "."
+    with open(input_file, "r") as fin:
+        with open(output_file, "w") as fout:
+            prev_line = ""
             for original_line in fin:
                 line = original_line
-                if 'lappend auto_path $env(PARFLOW_DIR)/bin' in line:
-                    line = 'from parflow import Run\n'
+                if "lappend auto_path $env(PARFLOW_DIR)/bin" in line:
+                    line = "from parflow import Run\n"
 
-                if 'package require parflow' in line:
-                    line = ''
+                if "package require parflow" in line:
+                    line = ""
 
-                if 'namespace import Parflow::*' in line:
+                if "namespace import Parflow::*" in line:
                     line = f'{run_name} = Run("{run_name}", __file__)\n'
 
-                if line[0:6] == 'pfset ':
-                    line = line.replace('pfset ', run_str)
+                if line[0:6] == "pfset ":
+                    line = line.replace("pfset ", run_str)
                     line_subs = line.split()
-                    line_subs[0] = line_subs[0].replace('-', '_')
-                    if (line_subs[1][0].isalpha() or
-                            line_subs[1][0] == "\""):
-                        line = (line_subs[0] + ' = ' + "'" +
-                                ' '.join(line_subs[1:]) + "'" + '\n')
-                        line = (line.replace('-', '_').replace('\"', '')
-                                .replace("'False'", "False")
-                                .replace("'True'", "True"))
-                    elif line_subs[1][0] == '$' and len(line_subs) == 2:
-                        line = line_subs[0] + ' = ' + line_subs[1][1:] + '\n'
+                    line_subs[0] = line_subs[0].replace("-", "_")
+                    if line_subs[1][0].isalpha() or line_subs[1][0] == '"':
+                        line = (
+                            line_subs[0]
+                            + " = "
+                            + "'"
+                            + " ".join(line_subs[1:])
+                            + "'"
+                            + "\n"
+                        )
+                        line = (
+                            line.replace("-", "_")
+                            .replace('"', "")
+                            .replace("'False'", "False")
+                            .replace("'True'", "True")
+                        )
+                    elif line_subs[1][0] == "$" and len(line_subs) == 2:
+                        line = line_subs[0] + " = " + line_subs[1][1:] + "\n"
                     else:
-                        line = (line_subs[0] + ' = ' +
-                                ' '.join(line_subs[1:]) + '\n')
+                        line = line_subs[0] + " = " + " ".join(line_subs[1:]) + "\n"
 
-                if line[0:4] == 'set ' and 'run_name' not in line:
-                    line = line.replace('set ', '')
+                if line[0:4] == "set " and "run_name" not in line:
+                    line = line.replace("set ", "")
                     line_subs = line.split()
                     if line_subs[1][0].isalpha():
-                        line = (line_subs[0] + ' = ' + "'" +
-                                ' '.join(line_subs[1:]) + "'" + '\n')
+                        line = (
+                            line_subs[0]
+                            + " = "
+                            + "'"
+                            + " ".join(line_subs[1:])
+                            + "'"
+                            + "\n"
+                        )
                     else:
-                        line = (line_subs[0] + ' = ' +
-                                ' '.join(line_subs[1:]) + '\n')
+                        line = line_subs[0] + " = " + " ".join(line_subs[1:]) + "\n"
 
                 # commenting out all lines of code that haven't been edited yet
-                if (line[0:1] != '#' and line[0:1] != '\n' and
-                        line == original_line):
+                if line[0:1] != "#" and line[0:1] != "\n" and line == original_line:
                     # testing for lines that continue to the next line
                     if len(prev_line) >= 2 and prev_line[-2] == "\\":
                         pass
                     else:
-                        line = '# ' + line
+                        line = "# " + line
 
                 prev_line = line
 
                 fout.write(line)
 
-            fout.write(f'{run_name}.run()\n')
+            fout.write(f"{run_name}.run()\n")
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description="Parflow TCL script converter")
-    parser.add_argument("--name", "-n",
-                        default=None,
-                        dest="name",
-                        help="Name of the run to use")
-    parser.add_argument("--input-file", "-i",
-                        default=None,
-                        dest="input",
-                        help="Path to ParFlow TCL script to convert")
-    parser.add_argument("--output-file", "-o",
-                        default=None,
-                        dest="output",
-                        help="Python file path to use for writting the "
-                             "converted input")
+    parser = argparse.ArgumentParser(description="Parflow TCL script converter")
+    parser.add_argument(
+        "--name", "-n", default=None, dest="name", help="Name of the run to use"
+    )
+    parser.add_argument(
+        "--input-file",
+        "-i",
+        default=None,
+        dest="input",
+        help="Path to ParFlow TCL script to convert",
+    )
+    parser.add_argument(
+        "--output-file",
+        "-o",
+        default=None,
+        dest="output",
+        help="Python file path to use for writting the " "converted input",
+    )
 
     args = parser.parse_args()
 
diff --git a/pftools/python/parflow/tools/__init__.py b/pftools/python/parflow/tools/__init__.py
index 23f88d68a..7d0f9ea52 100644
--- a/pftools/python/parflow/tools/__init__.py
+++ b/pftools/python/parflow/tools/__init__.py
@@ -6,11 +6,14 @@
 """
 from .core import Run
 from .io import ParflowBinaryReader, read_pfb, read_pfb_sequence, write_pfb
+from .compare import pf_test_file, pf_test_file_with_abs
 
 __all__ = [
-    'Run',
-    'ParflowBinaryReader',
-    'read_pfb',
-    'write_pfb',
-    'read_pfb_sequence',
+    "Run",
+    "ParflowBinaryReader",
+    "read_pfb",
+    "write_pfb",
+    "read_pfb_sequence",
+    "pf_test_file",
+    "pf_test_file_with_abs",
 ]
diff --git a/pftools/python/parflow/tools/builders.py b/pftools/python/parflow/tools/builders.py
index a0ec71ec8..c27487701 100644
--- a/pftools/python/parflow/tools/builders.py
+++ b/pftools/python/parflow/tools/builders.py
@@ -107,7 +107,16 @@ def side_ids(self, side_patch_ids):
         self.patch_ids_side = side_patch_ids
         return self
 
-    def write(self, name, xllcorner=0, yllcorner=0, cellsize=0, vtk=False, extra=None, generate_asc_files=False):
+    def write(
+        self,
+        name,
+        xllcorner=0,
+        yllcorner=0,
+        cellsize=0,
+        vtk=False,
+        extra=None,
+        generate_asc_files=False,
+    ):
         """Writing out pfsol file with optional output to vtk
 
         Args:
@@ -125,7 +134,7 @@ def write(self, name, xllcorner=0, yllcorner=0, cellsize=0, vtk=False, extra=Non
         self.name = name
         output_file_path = get_absolute_path(name)
         if self.mask_array is None:
-            raise Exception('No mask was defined')
+            raise Exception("No mask was defined")
 
         temp_pfb_file = None
         if generate_asc_files:
@@ -142,110 +151,123 @@ def write(self, name, xllcorner=0, yllcorner=0, cellsize=0, vtk=False, extra=Non
             for j in range(j_size):
                 for i in range(i_size):
                     if self.mask_array[j, i] != 0:
-                        patch_value = 0 if self.patch_ids_side is None \
+                        patch_value = (
+                            0
+                            if self.patch_ids_side is None
                             else self.patch_ids_side[j, i]
+                        )
                         # Left (-x)
                         if i == 0 or self.mask_array[j, i - 1] == 0:
-                            left_mask[j, i] = patch_value if patch_value \
-                                else self.side_id
+                            left_mask[j, i] = (
+                                patch_value if patch_value else self.side_id
+                            )
 
                         # Right (+x)
                         if i + 1 == i_size or self.mask_array[j, i + 1] == 0:
-                            right_mask[j, i] = patch_value if patch_value \
-                                else self.side_id
+                            right_mask[j, i] = (
+                                patch_value if patch_value else self.side_id
+                            )
 
                         # Back (-y) (y flipped)
                         if j + 1 == j_size or self.mask_array[j + 1, i] == 0:
-                            back_mask[j, i] = patch_value if patch_value \
-                                else self.side_id
+                            back_mask[j, i] = (
+                                patch_value if patch_value else self.side_id
+                            )
 
                         # Front (+y) (y flipped)
                         if j == 0 or self.mask_array[j - 1, i] == 0:
-                            front_mask[j, i] = patch_value if patch_value \
-                                else self.side_id
+                            front_mask[j, i] = (
+                                patch_value if patch_value else self.side_id
+                            )
 
                         # Bottom (-z)
-                        patch_value = 0 if self.patch_ids_bottom is None \
+                        patch_value = (
+                            0
+                            if self.patch_ids_bottom is None
                             else self.patch_ids_bottom[j, i]
-                        bottom_mask[j, i] = patch_value if patch_value \
-                            else self.bottom_id
+                        )
+                        bottom_mask[j, i] = (
+                            patch_value if patch_value else self.bottom_id
+                        )
 
                         # Top (+z)
-                        patch_value = 0 if self.patch_ids_top is None \
+                        patch_value = (
+                            0
+                            if self.patch_ids_top is None
                             else self.patch_ids_top[j, i]
-                        top_mask[j, i] = patch_value if patch_value \
-                            else self.top_id
+                        )
+                        top_mask[j, i] = patch_value if patch_value else self.top_id
 
             # Generate asc / sa files
             write_func = write_patch_matrix_as_asc
             settings = {
-                'xllcorner': xllcorner,
-                'yllcorner': yllcorner,
-                'cellsize': cellsize,
-                'NODATA_value': 0,
+                "xllcorner": xllcorner,
+                "yllcorner": yllcorner,
+                "cellsize": cellsize,
+                "NODATA_value": 0,
             }
             short_name = name[:-6]
 
-            left_file_path = get_absolute_path(f'{short_name}_left.asc')
+            left_file_path = get_absolute_path(f"{short_name}_left.asc")
             write_func(left_mask, left_file_path, **settings)
 
-            right_file_path = get_absolute_path(f'{short_name}_right.asc')
+            right_file_path = get_absolute_path(f"{short_name}_right.asc")
             write_func(right_mask, right_file_path, **settings)
 
-            front_file_path = get_absolute_path(f'{short_name}_front.asc')
+            front_file_path = get_absolute_path(f"{short_name}_front.asc")
             write_func(front_mask, front_file_path, **settings)
 
-            back_file_path = get_absolute_path(f'{short_name}_back.asc')
+            back_file_path = get_absolute_path(f"{short_name}_back.asc")
             write_func(back_mask, back_file_path, **settings)
 
-            top_file_path = get_absolute_path(f'{short_name}_top.asc')
+            top_file_path = get_absolute_path(f"{short_name}_top.asc")
             write_func(top_mask, top_file_path, **settings)
 
-            bottom_file_path = get_absolute_path(f'{short_name}_bottom.asc')
+            bottom_file_path = get_absolute_path(f"{short_name}_bottom.asc")
             write_func(bottom_mask, bottom_file_path, **settings)
 
             args = [
-                f'--mask-top {top_file_path}',
-                f'--mask-bottom {bottom_file_path}',
-                f'--mask-left {left_file_path}',
-                f'--mask-right {right_file_path}',
-                f'--mask-front {front_file_path}',
-                f'--mask-back {back_file_path}',
-                f'--pfsol {output_file_path}'
+                f"--mask-top {top_file_path}",
+                f"--mask-bottom {bottom_file_path}",
+                f"--mask-left {left_file_path}",
+                f"--mask-right {right_file_path}",
+                f"--mask-front {front_file_path}",
+                f"--mask-back {back_file_path}",
+                f"--pfsol {output_file_path}",
             ]
 
         else:
-            temp_pfb_file = tempfile.NamedTemporaryFile(suffix='.pfb')
+            temp_pfb_file = tempfile.NamedTemporaryFile(suffix=".pfb")
             if self.mask_array.dtype != np.float64:
                 self.mask_array = self.mask_array.astype(np.float64)
             write_pfb(temp_pfb_file.name, self.mask_array)
             args = [
-                f'--mask {temp_pfb_file.name}',
-                f'--side-patch-label {self.side_id}',
-                f'--bottom-patch-label {self.bottom_id}',
-                f'--pfsol {output_file_path}'
+                f"--mask {temp_pfb_file.name}",
+                f"--side-patch-label {self.side_id}",
+                f"--bottom-patch-label {self.bottom_id}",
+                f"--pfsol {output_file_path}",
             ]
 
         # Trigger conversion
-        print('=== pfmask-to-pfsol ===: BEGIN')
+        print("=== pfmask-to-pfsol ===: BEGIN")
         if extra is None:
             extra = []
         if vtk:
-            extra.append('--vtk')
-            extra.append(f'{output_file_path[:-6]}.vtk')
+            extra.append("--vtk")
+            extra.append(f"{output_file_path[:-6]}.vtk")
 
-        exe_path = get_absolute_path('$PARFLOW_DIR/bin/pfmask-to-pfsol')
+        exe_path = get_absolute_path("$PARFLOW_DIR/bin/pfmask-to-pfsol")
         args = args + extra
-        cmd_line = f'{exe_path} ' + ' '.join(args)
+        cmd_line = f"{exe_path} " + " ".join(args)
         process = Popen(cmd_line.split(), stdout=PIPE, stderr=PIPE)
         stdout, stderr = process.communicate()
-        print('Standard output:')
+        print("Standard output:")
         print(stdout)
-        print('')
-        print('Standard error:')
+        print("")
+        print("Standard error:")
         print(stderr)
 
-        print('=== pfmask-to-pfsol ===: END')
+        print("=== pfmask-to-pfsol ===: END")
 
         if temp_pfb_file is not None:
             temp_pfb_file.close()
@@ -259,10 +281,11 @@ def for_key(self, geom_item):
             geom_item (str): Name of geometric unit in ParFlow run that will
             bet used as a token for the ParFlow key.
         """
-        geom_item.InputType = 'SolidFile'
+        geom_item.InputType = "SolidFile"
         geom_item.FileName = self.name
         return self
 
+
 # -----------------------------------------------------------------------------
 # Abstract table to property helper class
 # -----------------------------------------------------------------------------
@@ -270,7 +293,7 @@ def for_key(self, geom_item):
 
 # splitting csv and txt lines into tokens
 def _csv_line_tokenizer(line):
-    return [token.strip() for token in line.split(',')]
+    return [token.strip() for token in line.split(",")]
 
 
 def _txt_line_tokenizer(line):
@@ -311,8 +334,9 @@ def __init__(self, run=None):
         self.column_index = {}
         self.props_in_row_header = True
         self.table_comments = []
+        self.empty_value_tokens = {"-", ""}
         yaml_key_def = Path(__file__).parent / self.reference_file
-        with open(yaml_key_def, 'r') as file:
+        with open(yaml_key_def, "r") as file:
             self.definition = yaml.safe_load(file)
 
         # Extract prop column names
@@ -321,8 +345,8 @@ def __init__(self, run=None):
         self.pfkey_to_alias = {}
         self.alias_duplicates = set()
         for key, value in self.definition.items():
-            self.pfkey_to_alias[key] = value['alias'][0]
-            for alias in value['alias']:
+            self.pfkey_to_alias[key] = value["alias"][0]
+            for alias in value["alias"]:
                 # checking for duplicate aliases
                 if alias in self.prop_names:
                     self.alias_duplicates.add(alias)
@@ -332,14 +356,14 @@ def __init__(self, run=None):
 
         # crashes if there are duplicate aliases
         if self.alias_duplicates:
-            raise Exception(f'Warning - duplicate alias name(s):'
-                            f' {self.alias_duplicates}')
+            raise Exception(
+                f"Warning - duplicate alias name(s):" f" {self.alias_duplicates}"
+            )
 
     def _process_data_line(self, tokens):
-        """Method to process lines of data in a table
-        """
+        """Method to process lines of data in a table"""
         # Skip new lines or comments
-        if not tokens or tokens[0] == '#':
+        if not tokens or tokens[0] == "#":
             return
 
         if self.props_in_row_header:
@@ -348,32 +372,32 @@ def _process_data_line(self, tokens):
             registrations = []
             for alias, col_idx in self.column_index.items():
                 str_value = tokens[col_idx]
-                if str_value == '-':
+                if str_value in self.empty_value_tokens:
                     continue
 
                 key = self.alias_to_pfkey[alias]
                 key_def = self.definition[key]
-                value_type = key_def.get('type', 'float')
+                value_type = key_def.get("type", "float")
                 value = __builtins__[value_type](str_value)
                 data[key] = value
 
                 # setting related addon keys
-                if 'addon' in key_def:
-                    for key, value in key_def['addon'].items():
+                if "addon" in key_def:
+                    for key, value in key_def["addon"].items():
                         # local keys (appending to geom item)
-                        if key.startswith('.'):
+                        if key.startswith("."):
                             data[key[1:]] = value
                         # global keys
                         elif key not in self.output:
                             self.output[key] = value
 
                 # appending geom name to list for setting geom name keys
-                if 'register' in key_def:
-                    registrations.append(key_def['register'])
+                if "register" in key_def:
+                    registrations.append(key_def["register"])
 
             # Extract unit_name
-            unit_name = data['key']
-            del data['key']
+            unit_name = data["key"]
+            del data["key"]
             self.output[unit_name] = data
 
             if not hasattr(self.name_registration, unit_name):
@@ -386,29 +410,29 @@ def _process_data_line(self, tokens):
             data = {}
             registrations = []
 
-            main_key = 'key'
+            main_key = "key"
             for key_alias in self.column_index:
-                if key_alias in self.definition['key']['alias']:
+                if key_alias in self.definition["key"]["alias"]:
                     main_key = key_alias
 
             prop_alias = tokens[self.column_index[main_key]]
             key = self.alias_to_pfkey[prop_alias]
             key_def = self.definition[key]
-            value_type = key_def.get('type', 'float')
+            value_type = key_def.get("type", "float")
             value_convert = __builtins__[value_type]
             # setting related addon keys
-            if 'addon' in key_def:
-                for addon_key, addon_value in key_def['addon'].items():
+            if "addon" in key_def:
+                for addon_key, addon_value in key_def["addon"].items():
                     # local keys (appending to geom item)
-                    if addon_key.startswith('.'):
+                    if addon_key.startswith("."):
                         data[addon_key[1:]] = addon_value
                     # global keys
                     elif addon_key not in self.output:
                         self.output[addon_key] = addon_value
 
             # appending geom name to list for setting geom name keys
-            if 'register' in key_def:
-                registrations.append(key_def['register'])
+            if "register" in key_def:
+                registrations.append(key_def["register"])
 
             for unit_name in self.column_index:
                 if unit_name == main_key:
@@ -416,7 +440,7 @@ def _process_data_line(self, tokens):
 
                 container = self.output[unit_name]
                 value_str = tokens[self.column_index[unit_name]]
-                if value_str == '-':
+                if value_str in self.empty_value_tokens:
                     continue
 
                 value = value_convert(value_str)
@@ -426,13 +450,12 @@ def _process_data_line(self, tokens):
                     self.name_registration[unit_name].update(registrations)
 
     def _process_first_line(self, first_line_tokens):
-        """Method to process first line in a table
-        """
+        """Method to process first line in a table"""
         # Skip new lines or comments
         if not first_line_tokens:
             return False
-        if first_line_tokens[0] == '#':
-            self.table_comments.append(' '.join(first_line_tokens))
+        if first_line_tokens[0] == "#":
+            self.table_comments.append(" ".join(first_line_tokens))
             return False
 
         self.props_in_row_header = None
@@ -449,31 +472,32 @@ def _process_first_line(self, first_line_tokens):
         if len(not_found) == 0:
             self.props_in_row_header = True
         elif len(found) > 1 and len(not_found) > 1:
-            print('Error while processing input table:')
-            print(f' - Properties found: {found}')
-            print(f' - Properties not found: {not_found}')
+            print("Error while processing input table:")
+            print(f" - Properties found: {found}")
+            print(f" - Properties not found: {not_found}")
         elif len(found) == 1:
             self.props_in_row_header = False
             # Prefill unit_name containers
             for unit_name in self.column_index:
-                if unit_name not in self.definition['key']['alias']:
+                if unit_name not in self.definition["key"]["alias"]:
                     self.output[unit_name] = {}
                     self.name_registration[unit_name] = set()
 
         if self.props_in_row_header is None:
-            raise Exception('Invalid table format')
+            print("Raising invalid table format, first line tokens are:\n")
+            print(first_line_tokens)
+            raise Exception("Invalid table format")
 
         return True
 
-    def load_csv_file(self, table_file, encoding='utf-8-sig'):
+    def load_csv_file(self, table_file, encoding="utf-8-sig"):
         """Method to load a .csv file of a table of parameters
 
         Args:
             table_file (str): Path to the input .csv file.
             encoding='utf-8-sig': encoding of input file.
         """
-        with open(get_absolute_path(table_file), 'r',
-                  encoding=encoding) as csv_file:
+        with open(get_absolute_path(table_file), "r", encoding=encoding) as csv_file:
             data_line = False
             for line in csv_file:
                 tokens = _csv_line_tokenizer(line)
@@ -483,15 +507,14 @@ def load_csv_file(self, table_file, encoding='utf-8-sig'):
                     data_line = self._process_first_line(tokens)
         return self
 
-    def load_txt_file(self, table_file, encoding='utf-8-sig'):
+    def load_txt_file(self, table_file, encoding="utf-8-sig"):
         """Method to load a .txt file of a table of parameters
 
         Args:
             table_file (str): Path to the input .txt file.
             encoding='utf-8-sig': encoding of input file.
         """
-        with open(get_absolute_path(table_file), 'r',
-                  encoding=encoding) as txt_file:
+        with open(get_absolute_path(table_file), "r", encoding=encoding) as txt_file:
             data_line = False
             for line in txt_file:
                 tokens = _txt_line_tokenizer(line)
@@ -560,30 +583,31 @@ def load_default_properties(self, database=None):
         if database is None:
             database = self.default_db
 
-        database_file = f'ref/{self.db_prefix}{database}.txt'
+        database_file = f"ref/{self.db_prefix}{database}.txt"
         default_prop_file = str(Path(__file__).parent / database_file)
 
         if exists(default_prop_file):
             self.load_txt_file(default_prop_file)
-            print('#' * 80)
-            print('# Loaded database:')
+            print("#" * 80)
+            print("# Loaded database:")
             for item in self.table_comments:
                 print(item)
-            print('#' * 80)
+            print("#" * 80)
         else:
-            print('#' * 80)
-            print(f'# {database} database not found. Available databases '
-                  f'include:')
-            for root, dirs, files in os.walk(Path(__file__).parent / 'ref'):
+            print("#" * 80)
+            print(f"# {database} database not found. Available databases " f"include:")
+            for root, dirs, files in os.walk(Path(__file__).parent / "ref"):
                 for name in files:
                     if name.startswith(self.db_prefix):
-                        print(f'# - {name} (use argument '
-                              f'"{remove_prefix(name, self.db_prefix)}")')
-            print('#' * 80)
+                        print(
+                            f"# - {name} (use argument "
+                            f'"{remove_prefix(name, self.db_prefix)}")'
+                        )
+            print("#" * 80)
 
         return self
 
-    def apply(self, run=None, name_registration=True):
+    def apply(self, run=None, name_registration=True, infer_key_names=False):
         """Method to apply the loaded properties to a given
            run object.
 
@@ -598,11 +622,19 @@ def apply(self, run=None, name_registration=True):
         # applying subsurface properties to run keys
         if run is None:
             if self.run is None:
-                print('No run object assigned')
+                print("No run object assigned")
                 sys.exit(1)
         else:
             self.run = run
 
+        # new names are user specified names such as s1 s2 s3 for soil permeability
+        if infer_key_names and name_registration:
+            new_names = ""
+            for name in self.output:
+                if isinstance(self.output[name], dict):
+                    new_names = new_names + name + " "
+            self.key_root.Names = new_names
+
         valid_unit_names = []
         addon_keys = {}
         for name in self.output:
@@ -619,9 +651,13 @@ def apply(self, run=None, name_registration=True):
         if name_registration:
             names_to_set = addon_keys
             for unit_name in valid_unit_names:
-                if unit_name in self.name_registration:
+                if unit_name in (
+                    registered_name for registered_name in self.name_registration
+                ):
                     for prop_name in self.name_registration[unit_name]:
-                        if prop_name not in names_to_set:
+                        if prop_name not in (
+                            name_to_set for name_to_set in names_to_set
+                        ):
                             names_to_set[prop_name] = []
                         names_to_set[prop_name].append(unit_name)
             self.run.pfset(flat_map=names_to_set)
@@ -629,8 +665,7 @@ def apply(self, run=None, name_registration=True):
         return self
 
     def print(self):
-        """Method to print subsurface properties in hierarchical format
-        """
+        """Method to print subsurface properties in hierarchical format"""
         output_to_print = {self.unit_string: {}}
         valid_unit_names = []
         for unit_name in self.output:
@@ -649,7 +684,7 @@ def print(self):
         print(yaml.dump(sort_dict(output_to_print), Dumper=NoAliasDumper))
         return self
 
-    def get_table(self, props_in_header=True, column_separator='  '):
+    def get_table(self, props_in_header=True, column_separator="  "):
         """Method to convert loaded subsurface properties into a table
 
         Args:
@@ -663,8 +698,8 @@ def get_table(self, props_in_header=True, column_separator='  '):
         """
         entries = []
         prop_set = set()
-        prop_sizes = {'key': 0}
-        unit_sizes = {'key': 0}
+        prop_sizes = {"key": 0}
+        unit_sizes = {"key": 0}
 
         # Fill entries headers
         for unit_name, props in self.output.items():
@@ -674,9 +709,9 @@ def get_table(self, props_in_header=True, column_separator='  '):
                 continue
 
             entry = {
-                'key': unit_name,
+                "key": unit_name,
             }
-            prop_sizes['key'] = max(prop_sizes['key'], len(unit_name))
+            prop_sizes["key"] = max(prop_sizes["key"], len(unit_name))
             unit_sizes[unit_name] = len(unit_name)
             for prop in props:
                 if prop in self.pfkey_to_alias:
@@ -695,7 +730,7 @@ def get_table(self, props_in_header=True, column_separator='  '):
                     else:
                         prop_sizes[alias] = max(prop_sizes[alias], size)
 
-                    unit_sizes['key'] = max(unit_sizes['key'], len(alias))
+                    unit_sizes["key"] = max(unit_sizes["key"], len(alias))
 
             entries.append(entry)
 
@@ -727,7 +762,7 @@ def get_table(self, props_in_header=True, column_separator='  '):
             for entry in entries:
                 line = []
                 for key in header_keys:
-                    value = entry[key] if key in entry else '-'
+                    value = entry[key] if key in entry else "-"
                     width = sizes[key]
                     line.append(value.ljust(width))
                 table_lines.append(column_separator.join(line))
@@ -748,25 +783,24 @@ def get_table(self, props_in_header=True, column_separator='  '):
             for prop in prop_set:
                 line = []
                 for key in header_keys:
-                    if key == 'key':
+                    if key == "key":
                         width = sizes[key]
                         line.append(prop.ljust(width))
                         continue
 
                     for entry in entries:
-                        if entry['key'] != key:
+                        if entry["key"] != key:
                             continue
-                        value = entry[prop] if prop in entry else '-'
+                        value = entry[prop] if prop in entry else "-"
                         width = sizes[key]
                         line.append(value.ljust(width))
 
                 table_lines.append(column_separator.join(line))
 
-        return '\n'.join(table_lines)
+        return "\n".join(table_lines)
 
-    def print_as_table(self, props_in_header=True, column_separator='  '):
-        """Method to print the table returned from the get_table method
-        """
+    def print_as_table(self, props_in_header=True, column_separator="  "):
+        """Method to print the table returned from the get_table method"""
         print(self.get_table(props_in_header, column_separator))
         return self
 
@@ -775,6 +809,7 @@ def print_as_table(self, props_in_header=True, column_separator='  '):
 # Subsurface hydraulic property input helper
 # -----------------------------------------------------------------------------
 
+
 class SubsurfacePropertiesBuilder(TableToProperties):
 
     def __init__(self, run=None):
@@ -782,7 +817,7 @@ def __init__(self, run=None):
 
     @property
     def reference_file(self):
-        return 'ref/table_keys.yaml'
+        return "ref/table_keys.yaml"
 
     @property
     def key_root(self):
@@ -790,21 +825,48 @@ def key_root(self):
 
     @property
     def unit_string(self):
-        return 'Geom'
+        return "Geom"
 
     @property
     def default_db(self):
-        return 'conus_1'
+        return "conus_1"
 
     @property
     def db_prefix(self):
-        return 'subsurface_'
+        return "subsurface_"
+
+
+class ReservoirPropertiesBuilder(TableToProperties):
+
+    def __init__(self, run=None):
+        super().__init__(run)
+
+    @property
+    def reference_file(self):
+        return "ref/reservoir_keys.yaml"
+
+    @property
+    def key_root(self):
+        return self.run.Reservoirs
+
+    @property
+    def unit_string(self):
+        return "Reservoirs"
+
+    @property
+    def default_db(self):
+        return "conus_1"
+
+    @property
+    def db_prefix(self):
+        return "reservoirs_"
 
 
 # -----------------------------------------------------------------------------
 # Vegetation parameter property input helper
 # -----------------------------------------------------------------------------
 
+
 class VegParamBuilder(TableToProperties):
 
     def __init__(self, run=None):
@@ -812,7 +874,7 @@ def __init__(self, run=None):
 
     @property
     def reference_file(self):
-        return 'ref/vegp_keys.yaml'
+        return "ref/vegp_keys.yaml"
 
     @property
     def key_root(self):
@@ -820,15 +882,16 @@ def key_root(self):
 
     @property
     def unit_string(self):
-        return 'VegParams'
+        return "VegParams"
 
     @property
     def default_db(self):
-        return 'igbp'
+        return "igbp"
 
     @property
     def db_prefix(self):
-        return 'vegp_'
+        return "vegp_"
+
 
 # -----------------------------------------------------------------------------
 # Domain input builder - setting keys for various common problem definitions
@@ -837,37 +900,35 @@ def db_prefix(self):
 
 class DomainBuilder:
 
-    def __init__(self, run, name='domain'):
+    def __init__(self, run, name="domain"):
         self.run = run
         self.run.Domain.GeomName = name
 
     def __file_check(self, file_name, key_path):
-        """Checking files and setting keys for FileName keys
-        """
-        container, = self.run.select(key_path)
+        """Checking files and setting keys for FileName keys"""
+        (container,) = self.run.select(key_path)
         container.FileName = file_name
         ext = Path(file_name).suffix
-        if ext == '.pfb':
-            container.Type = 'PFBFile'
-        elif ext == '.nc':
-            container.Type = 'NCFile'
+        if ext == ".pfb":
+            container.Type = "PFBFile"
+        elif ext == ".nc":
+            container.Type = "NCFile"
         else:
-            raise Exception(f'File extension {ext} for {file_name} is invalid')
+            raise Exception(f"File extension {ext} for {file_name} is invalid")
 
         return self
 
     def water(self, geom_name=None):
-        """Setting keys for water properties and gravity
-        """
+        """Setting keys for water properties and gravity"""
         self.run.Gravity = 1.0
-        self.run.Phase.Names = 'water'
-        self.run.Phase.water.Density.Type = 'Constant'
+        self.run.Phase.Names = "water"
+        self.run.Phase.water.Density.Type = "Constant"
         self.run.Phase.water.Density.Value = 1.0
-        self.run.Phase.water.Viscosity.Type = 'Constant'
+        self.run.Phase.water.Viscosity.Type = "Constant"
         self.run.Phase.water.Viscosity.Value = 1.0
-        self.run.Phase.water.Mobility.Type = 'Constant'
+        self.run.Phase.water.Mobility.Type = "Constant"
         self.run.Phase.water.Mobility.Value = 1.0
-        self.run.PhaseSources.water.Type = 'Constant'
+        self.run.PhaseSources.water.Type = "Constant"
 
         if geom_name:
             self.run.PhaseSources.water.GeomNames = geom_name
@@ -876,16 +937,14 @@ def water(self, geom_name=None):
         return self
 
     def no_wells(self):
-        """Setting key with no wells
-        """
-        self.run.Wells.Names = ''
+        """Setting key with no wells"""
+        self.run.Wells.Names = ""
 
         return self
 
     def no_contaminants(self):
-        """Setting key with no contaminants
-        """
-        self.run.Contaminants.Names = ''
+        """Setting key with no contaminants"""
+        self.run.Contaminants.Names = ""
 
         return self
 
@@ -893,14 +952,14 @@ def variably_saturated(self):
         """Setting keys for variably saturated domain.
         Common solver settings
         """
-        self.run.Solver = 'Richards'
+        self.run.Solver = "Richards"
         self.run.Solver.Nonlinear.MaxIter = 10
         self.run.Solver.Nonlinear.ResidualTol = 1e-5
-        self.run.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+        self.run.Solver.Nonlinear.EtaChoice = "EtaConstant"
         self.run.Solver.Nonlinear.EtaValue = 1e-5
         self.run.Solver.Nonlinear.UseJacobian = True
         self.run.Solver.Nonlinear.DerivativeEpsilon = 1e-2
-        self.run.Solver.Linear.Preconditioner = 'PFMG'
+        self.run.Solver.Linear.Preconditioner = "PFMG"
 
         return self
 
@@ -908,86 +967,93 @@ def fully_saturated(self):
         """Fully saturated solver settings (other than solver ='Impes')
         Taken from default Impes examples - need to change
         """
-        self.run.Solver = 'Impes'
+        self.run.Solver = "Impes"
         return self
 
-    def homogeneous_subsurface(self, domain_name, perm=None, porosity=None,
-                               specific_storage=None, rel_perm=None,
-                               saturation=None, isotropic=False):
-        """Setting constant parameters for homogeneous subsurface
-        """
+    def homogeneous_subsurface(
+        self,
+        domain_name,
+        perm=None,
+        porosity=None,
+        specific_storage=None,
+        rel_perm=None,
+        saturation=None,
+        isotropic=False,
+    ):
+        """Setting constant parameters for homogeneous subsurface"""
         if perm is not None:
             if not self.run.Geom.Perm.Names:
                 self.run.Geom.Perm.Names = []
 
-            self.run.Geom.Perm._details_['Names']['history'] = []
+            self.run.Geom.Perm._details_["Names"]["history"] = []
             self.run.Geom.Perm.Names += [domain_name]
 
             # checking for Perm file
             if isinstance(perm, str):
-                self.__file_check(perm, f'Geom/{domain_name}/Perm')
+                self.__file_check(perm, f"Geom/{domain_name}/Perm")
             else:
-                self.run.Geom[domain_name].Perm.Type = 'Constant'
+                self.run.Geom[domain_name].Perm.Type = "Constant"
                 self.run.Geom[domain_name].Perm.Value = perm
 
         if porosity is not None:
             if not self.run.Geom.Porosity.GeomNames:
                 self.run.Geom.Porosity.GeomNames = []
 
-            self.run.Geom.Porosity._details_['GeomNames']['history'] = []
+            self.run.Geom.Porosity._details_["GeomNames"]["history"] = []
             self.run.Geom.Porosity.GeomNames += [domain_name]
 
             # checking for Porosity file
             if isinstance(porosity, str):
-                self.__file_check(porosity, f'Geom/{domain_name}/Porosity')
+                self.__file_check(porosity, f"Geom/{domain_name}/Porosity")
             else:
-                self.run.Geom[domain_name].Porosity.Type = 'Constant'
+                self.run.Geom[domain_name].Porosity.Type = "Constant"
                 self.run.Geom[domain_name].Porosity.Value = porosity
 
         if specific_storage is not None:
             if not self.run.SpecificStorage.GeomNames:
                 self.run.SpecificStorage.GeomNames = []
 
-            self.run.SpecificStorage._details_['GeomNames']['history'] = []
+            self.run.SpecificStorage._details_["GeomNames"]["history"] = []
             self.run.SpecificStorage.GeomNames += [domain_name]
-            self.run.SpecificStorage.Type = 'Constant'
+            self.run.SpecificStorage.Type = "Constant"
             self.run.Geom[domain_name].SpecificStorage.Value = specific_storage
 
         if rel_perm is not None:
             if not self.run.Phase.RelPerm.GeomNames:
                 self.run.Phase.RelPerm.GeomNames = []
 
-            self.run.Phase.RelPerm.Type = rel_perm['Type']
-            self.run.Phase.RelPerm._details_['GeomNames']['history'] = []
+            self.run.Phase.RelPerm.Type = rel_perm["Type"]
+            self.run.Phase.RelPerm._details_["GeomNames"]["history"] = []
             self.run.Phase.RelPerm.GeomNames += [domain_name]
-            if rel_perm['Type'] == 'VanGenuchten':
-                self.run.Geom[domain_name].RelPerm.Alpha = rel_perm['Alpha']
-                self.run.Geom[domain_name].RelPerm.N = rel_perm['N']
+            if rel_perm["Type"] == "VanGenuchten":
+                self.run.Geom[domain_name].RelPerm.Alpha = rel_perm["Alpha"]
+                self.run.Geom[domain_name].RelPerm.N = rel_perm["N"]
 
         if saturation is not None:
             if not self.run.Phase.Saturation.GeomNames:
                 self.run.Phase.Saturation.GeomNames = []
 
-            self.run.Phase.Saturation.Type = saturation['Type']
-            self.run.Phase.Saturation._details_['GeomNames']['history'] = []
+            self.run.Phase.Saturation.Type = saturation["Type"]
+            self.run.Phase.Saturation._details_["GeomNames"]["history"] = []
             self.run.Phase.Saturation.GeomNames += [domain_name]
-            if saturation['Type'] == 'VanGenuchten':
+            if saturation["Type"] == "VanGenuchten":
                 # defaulting to RelPerm not working
                 self.run.Geom[domain_name].Saturation.Alpha = (
-                    saturation['Alpha'] if saturation['Alpha']
-                    else rel_perm['Alpha'])
+                    saturation["Alpha"] if saturation["Alpha"] else rel_perm["Alpha"]
+                )
                 self.run.Geom[domain_name].Saturation.N = (
-                    saturation['N'] if saturation['N'] else rel_perm['N'])
-                self.run.Geom[domain_name].Saturation.SRes = saturation['SRes']
-                self.run.Geom[domain_name].Saturation.SSat = saturation['SSat']
+                    saturation["N"] if saturation["N"] else rel_perm["N"]
+                )
+                self.run.Geom[domain_name].Saturation.SRes = saturation["SRes"]
+                self.run.Geom[domain_name].Saturation.SSat = saturation["SSat"]
 
         if isotropic:
-            self.run.Perm.TensorType = 'TensorByGeom'
+            self.run.Perm.TensorType = "TensorByGeom"
 
             if not self.run.Geom.Perm.TensorByGeom.Names:
                 self.run.Geom.Perm.TensorByGeom.Names = []
 
-            self.run.Geom.Perm.TensorByGeom._details_['Names']['history'] = []
+            self.run.Geom.Perm.TensorByGeom._details_["Names"]["history"] = []
             self.run.Geom.Perm.TensorByGeom.Names += [domain_name]
             self.run.Geom[domain_name].Perm.TensorValX = 1.0
             self.run.Geom[domain_name].Perm.TensorValY = 1.0
@@ -995,27 +1061,25 @@ def homogeneous_subsurface(self, domain_name, perm=None, porosity=None,
 
         return self
 
-    def box_domain(self, box_input, domain_geom_name,
-                   bounds=None, patches=None):
-        """Defining box domain and extents
-        """
+    def box_domain(self, box_input, domain_geom_name, bounds=None, patches=None):
+        """Defining box domain and extents"""
 
         if not self.run.GeomInput.Names:
             self.run.GeomInput.Names = []
 
         if box_input not in self.run.GeomInput.Names:
-            self.run.GeomInput._details_['Names']['history'] = []
+            self.run.GeomInput._details_["Names"]["history"] = []
             self.run.GeomInput.Names += [box_input]
 
         box_input_obj = self.run.GeomInput[box_input]
         if not box_input_obj.InputType:
-            box_input_obj.InputType = 'Box'
+            box_input_obj.InputType = "Box"
 
         if not box_input_obj.GeomName:
             box_input_obj.GeomName = []
 
         if domain_geom_name not in box_input_obj.GeomName:
-            box_input_obj._details_['GeomName']['history'] = []
+            box_input_obj._details_["GeomName"]["history"] = []
             box_input_obj.GeomName += [domain_geom_name]
 
         domain_geom = self.run.Geom[domain_geom_name]
@@ -1040,79 +1104,77 @@ def box_domain(self, box_input, domain_geom_name,
 
         return self
 
-    def slopes_mannings(self, domain_geom_name, slope_x=None,
-                        slope_y=None, mannings=None):
+    def slopes_mannings(
+        self, domain_geom_name, slope_x=None, slope_y=None, mannings=None
+    ):
         """Setting slopes and mannings coefficients as constant value
         or from an external file
         """
         if slope_x is not None:
             self.run.TopoSlopesX.GeomNames = domain_geom_name
             if isinstance(slope_x, str):
-                self.__file_check(slope_x, 'TopoSlopesX')
+                self.__file_check(slope_x, "TopoSlopesX")
             else:
-                self.run.TopoSlopesX.Type = 'Constant'
+                self.run.TopoSlopesX.Type = "Constant"
                 self.run.TopoSlopesX.Geom[domain_geom_name].Value = slope_x
         if slope_y is not None:
             self.run.TopoSlopesY.GeomNames = domain_geom_name
             if isinstance(slope_y, str):
-                self.__file_check(slope_y, 'TopoSlopesY')
+                self.__file_check(slope_y, "TopoSlopesY")
             else:
-                self.run.TopoSlopesY.Type = 'Constant'
+                self.run.TopoSlopesY.Type = "Constant"
                 self.run.TopoSlopesY.Geom[domain_geom_name].Value = slope_y
         if mannings is not None:
             self.run.Mannings.GeomNames = domain_geom_name
             if isinstance(mannings, str):
-                self.__file_check(mannings, 'Mannings')
+                self.__file_check(mannings, "Mannings")
             else:
-                self.run.Mannings.Type = 'Constant'
+                self.run.Mannings.Type = "Constant"
                 self.run.Mannings.Geom[domain_geom_name].Value = mannings
 
         return self
 
     def zero_flux(self, patches, cycle_name, interval_name):
-        """Setting zero-flux boundary condition for patch or patches
-        """
+        """Setting zero-flux boundary condition for patch or patches"""
         if not self.run.BCPressure.PatchNames:
             self.run.BCPressure.PatchNames = []
 
         for patch in patches.split():
-            self.run.BCPressure._details_['PatchNames']['history'] = []
+            self.run.BCPressure._details_["PatchNames"]["history"] = []
             self.run.BCPressure.PatchNames += [patch]
-            self.run.Patch[patch].BCPressure.Type = 'FluxConst'
+            self.run.Patch[patch].BCPressure.Type = "FluxConst"
             self.run.Patch[patch].BCPressure.Cycle = cycle_name
             self.run.Patch[patch].BCPressure[interval_name].Value = 0.0
 
         return self
 
     def ic_pressure(self, domain_geom_name, patch, pressure):
-        """Setting initial condition pressure from file or to constant value
-        """
+        """Setting initial condition pressure from file or to constant value"""
         self.run.ICPressure.GeomNames = domain_geom_name
         self.run.Geom[domain_geom_name].ICPressure.RefPatch = patch
 
-        if isinstance(pressure, str) and Path(pressure).suffix == '.pfb':
-            self.run.ICPressure.Type = 'PFBFile'
+        if isinstance(pressure, str) and Path(pressure).suffix == ".pfb":
+            self.run.ICPressure.Type = "PFBFile"
             self.run.Geom.domain.ICPressure.FileName = pressure
         elif isinstance(pressure, (float, int)):
-            self.run.ICPressure.Type = 'HydroStaticPatch'
+            self.run.ICPressure.Type = "HydroStaticPatch"
             self.run.Geom.domain.ICPressure.Value = pressure
         else:
-            raise Exception(f'Incompatible type or file of {pressure}')
+            raise Exception(f"Incompatible type or file of {pressure}")
 
         return self
 
     def clm(self, met_file_name, top_patch, cycle_name, interval_name):
-        """Setting keys associated with CLM
-        """
+        """Setting keys associated with CLM"""
         # ensure time step is hourly
-        self.run.TimeStep.Type = 'Constant'
+        self.run.TimeStep.Type = "Constant"
         self.run.TimeStep.Value = 1.0
         # ensure OverlandFlow is the top boundary condition
-        self.run.Patch[top_patch].BCPressure.Type = 'OverlandFlow'
+        self.run.Patch[top_patch].BCPressure.Type = "OverlandFlow"
         self.run.Patch[top_patch].BCPressure.Cycle = cycle_name
         self.run.Patch[top_patch].BCPressure[interval_name].Value = 0.0
         # set CLM keys
-        self.run.Solver.LSM = 'CLM'
+        self.run.Solver.LSM = "CLM"
         self.run.Solver.CLM.CLMFileDir = "."
         self.run.Solver.PrintCLM = True
         self.run.Solver.CLM.Print1dOut = False
@@ -1122,73 +1184,120 @@ def clm(self, met_file_name, top_patch, cycle_name, interval_name):
         self.run.Solver.CLM.CLMDumpInterval = 24
         self.run.Solver.CLM.WriteLogs = False
         self.run.Solver.CLM.WriteLastRST = True
-        self.run.Solver.CLM.MetForcing = '1D'
+        self.run.Solver.CLM.MetForcing = "1D"
         self.run.Solver.CLM.MetFileName = met_file_name
         self.run.Solver.CLM.MetFilePath = "."
         self.run.Solver.CLM.MetFileNT = 24
         self.run.Solver.CLM.IstepStart = 1.0
-        self.run.Solver.CLM.EvapBeta = 'Linear'
-        self.run.Solver.CLM.VegWaterStress = 'Saturation'
+        self.run.Solver.CLM.EvapBeta = "Linear"
+        self.run.Solver.CLM.VegWaterStress = "Saturation"
         self.run.Solver.CLM.ResSat = 0.1
         self.run.Solver.CLM.WiltingPoint = 0.12
         self.run.Solver.CLM.FieldCapacity = 0.98
-        self.run.Solver.CLM.IrrigationType = 'none'
+        self.run.Solver.CLM.IrrigationType = "none"
 
         return self
 
-    def well(self, name, type, x, y, z_upper, z_lower,
-             cycle_name, interval_name, action='Extraction',
-             saturation=1.0, phase='water', hydrostatic_pressure=None,
-             value=None):
-        """Setting keys necessary to define a simple well
-        """
+    def reservoir(
+        self,
+        name,
+        Intake_X,
+        Intake_Y,
+        Has_Secondary_Intake_Cell,
+        Secondary_Intake_X,
+        Secondary_Intake_Y,
+        Release_X,
+        Release_Y,
+        Release_Rate,
+        Max_Storage,
+        Storage,
+        Min_Release_Storage,
+        Overland_Flow_Solver,
+    ):
+        """Setting keys necessary to define a simple reservoir"""
+
+        if not self.run.Reservoirs.Names:
+            self.run.Reservoirs.Names = []
+
+        self.run.Reservoirs.Names += [name]
+        reservoir = self.run.Reservoirs[name]
+        reservoir.Intake_X = Intake_X
+        reservoir.Intake_Y = Intake_Y
+        reservoir.Secondary_Intake_X = Secondary_Intake_X
+        reservoir.Secondary_Intake_Y = Secondary_Intake_Y
+        reservoir.Has_Secondary_Intake_Cell = Has_Secondary_Intake_Cell
+        reservoir.Overland_Flow_Solver = Overland_Flow_Solver
+        reservoir.Release_X = Release_X
+        reservoir.Release_Y = Release_Y
+        reservoir.Release_Rate = Release_Rate
+        reservoir.Max_Storage = Max_Storage
+        reservoir.Storage = Storage
+        reservoir.Min_Release_Storage = Min_Release_Storage
+        return self
+
+    def well(
+        self,
+        name,
+        type,
+        x,
+        y,
+        z_upper,
+        z_lower,
+        cycle_name,
+        interval_name,
+        action="Extraction",
+        saturation=1.0,
+        phase="water",
+        hydrostatic_pressure=None,
+        value=None,
+    ):
+        """Setting keys necessary to define a simple well"""
 
         if not self.run.Wells.Names:
             self.run.Wells.Names = []
 
         self.run.Wells.Names += [name]
         well = self.run.Wells[name]
-        well.InputType = 'Vertical'
-        well.Action = 'Extraction'
+        well.InputType = "Vertical"
+        well.Action = "Extraction"
         well.Type = type
         well.X = x
         well.Y = y
         well.ZUpper = z_upper
         well.ZLower = z_lower
-        well.Method = 'Standard'
+        well.Method = "Standard"
         well.Cycle = cycle_name
         well[interval_name].Saturation[phase].Value = saturation
 
-        if action == 'Extraction':
-            well.Action = 'Extraction'
-            if type == 'Pressure':
+        if action == "Extraction":
+            well.Action = "Extraction"
+            if type == "Pressure":
                 well[interval_name].Pressure.Value = hydrostatic_pressure
                 if value is not None:
                     well[interval_name].Extraction.Pressure.Value = value
-            elif type == 'Flux' and value is not None:
+            elif type == "Flux" and value is not None:
                 well[interval_name].Extraction.Flux[phase].Value = value
 
-        if action == 'Injection':
-            well.Action = 'Injection'
-            if type == 'Pressure':
+        if action == "Injection":
+            well.Action = "Injection"
+            if type == "Pressure":
                 well[interval_name].Pressure.Value = hydrostatic_pressure
                 if value is not None:
                     well[interval_name].Injection.Pressure.Value = value
-            elif type == 'Flux' and value is not None:
+            elif type == "Flux" and value is not None:
                 well[interval_name].Injection.Flux[phase].Value = value
 
         return self
 
     def spinup_timing(self, initial_step, dump_interval):
-        """Setting keys to assist a spinup run
-        """
+        """Setting keys to assist a spinup run"""
 
         self.run.TimingInfo.BaseUnit = 1
         self.run.TimingInfo.StartCount = 0
         self.run.TimingInfo.StartTime = 0.0
         self.run.TimingInfo.StopTime = 10000000
         self.run.TimingInfo.DumpInterval = dump_interval
-        self.run.TimeStep.Type = 'Growth'
+        self.run.TimeStep.Type = "Growth"
         self.run.TimeStep.InitialStep = initial_step
         self.run.TimeStep.GrowthFactor = 1.1
         self.run.TimeStep.MaxStep = 1000000
@@ -1196,22 +1305,49 @@ def spinup_timing(self, initial_step, dump_interval):
 
         return self
 
-    def clm_input(self, StartDate, StartTime, EndDate, EndTime,
-                  metf1d, outf1d, poutf1d, rstf, startcode=2,
-                  clm_ic=2, maxt=1, mina=0.05, udef=-9999, vclass=2,
-                  vegtf='drv_vegm.dat', vegpf='drv_vegp.dat',
-                  t_ini=300, h2osno_ini=0, surfind=2, soilind=1,
-                  snowind=0, forc_hgt_u=10.0, forc_hgt_t=2.0,
-                  forc_hgt_q=2.0, dewmx=0.1, qflx_tran_vegmx=-9999.0,
-                  rootfr=-9999.0, zlnd=0.01, zsno=0.0024,
-                  csoilc=0.0025, capr=0.34, cnfac=0.5, smpmin=-1.0e8,
-                  ssi=0.033, wimp=0.05):
-        """Setting metadata keys to build the CLM driver input file
-        """
-        syear, smonth, sday = StartDate.split('-')
-        shour, smin, ssec = StartTime.split('-')
-        eyear, emonth, eday = EndDate.split('-')
-        ehour, emin, esec = EndTime.split('-')
+    def clm_input(
+        self,
+        StartDate,
+        StartTime,
+        EndDate,
+        EndTime,
+        metf1d,
+        outf1d,
+        poutf1d,
+        rstf,
+        startcode=2,
+        clm_ic=2,
+        maxt=1,
+        mina=0.05,
+        udef=-9999,
+        vclass=2,
+        vegtf="drv_vegm.dat",
+        vegpf="drv_vegp.dat",
+        t_ini=300,
+        h2osno_ini=0,
+        surfind=2,
+        soilind=1,
+        snowind=0,
+        forc_hgt_u=10.0,
+        forc_hgt_t=2.0,
+        forc_hgt_q=2.0,
+        dewmx=0.1,
+        qflx_tran_vegmx=-9999.0,
+        rootfr=-9999.0,
+        zlnd=0.01,
+        zsno=0.0024,
+        csoilc=0.0025,
+        capr=0.34,
+        cnfac=0.5,
+        smpmin=-1.0e8,
+        ssi=0.033,
+        wimp=0.05,
+    ):
+        """Setting metadata keys to build the CLM driver input file"""
+        syear, smonth, sday = StartDate.split("-")
+        shour, smin, ssec = StartTime.split("-")
+        eyear, emonth, eday = EndDate.split("-")
+        ehour, emin, esec = EndTime.split("-")
 
         self.run.Solver.CLM.Input.Timing.StartYear = syear
         self.run.Solver.CLM.Input.Timing.StartMonth = smonth
@@ -1264,17 +1400,17 @@ class CLMImporter:
     def __init__(self, run):
         self.run = run
 
-    def files(self, input='drv_clmin.dat', map=None, parameters=None):
+    def files(self, input="drv_clmin.dat", map=None, parameters=None):
 
         def load_file(func_name, file_name):
             func = getattr(self, func_name)
             try:
                 func(file_name)
             except FileNotFoundError:
-                raise Exception(f'Could not find CLM driver file {file_name}')
+                raise Exception(f"Could not find CLM driver file {file_name}")
 
         # Load the clmin file first to determine the driver file names.
-        load_file('input_file', input)
+        load_file("input_file", input)
 
         if map is None:
             map = self._map_file_name
@@ -1282,15 +1418,15 @@ def load_file(func_name, file_name):
         if parameters is None:
             parameters = self._parameters_file_name
 
-        load_file('map_file', map)
-        load_file('parameters_file', parameters)
+        load_file("map_file", map)
+        load_file("parameters_file", parameters)
         return self
 
     @with_absolute_path
-    def input_file(self, path='drv_clmin.dat'):
-        clm_key_dict = read_clm(path, type='clmin')
+    def input_file(self, path="drv_clmin.dat"):
+        clm_key_dict = read_clm(path, type="clmin")
         self.input(clm_key_dict)
-        self._import_paths['input'] = path
+        self._import_paths["input"] = path
         return self
 
     def input(self, clm_key_dict):
@@ -1302,7 +1438,7 @@ def input(self, clm_key_dict):
                 invalid_keys.append(key)
                 continue
 
-            key_to_set = '.'.join(ref_dict[key])
+            key_to_set = ".".join(ref_dict[key])
             # Ensure the value is the right type...
             value = self._convert_to_domain_type(key_to_set, value)
             self.run.pfset(key=key_to_set, value=value)
@@ -1311,17 +1447,17 @@ def input(self, clm_key_dict):
         self._clear_all_histories(self.run.Solver.CLM.Input)
 
         if invalid_keys:
-            print('Warning: The following CLM variables could not be set:')
+            print("Warning: The following CLM variables could not be set:")
             for var in invalid_keys:
-                print(f'  - {var}')
+                print(f"  - {var}")
 
         return self
 
     @with_absolute_path
-    def parameters_file(self, path='drv_vegp.dat'):
-        vegp_data = read_clm(path, type='vegp')
+    def parameters_file(self, path="drv_vegp.dat"):
+        vegp_data = read_clm(path, type="vegp")
         self.parameters(vegp_data)
-        self._import_paths['parameters'] = path
+        self._import_paths["parameters"] = path
         return self
 
     def parameters(self, vegp_data):
@@ -1329,38 +1465,32 @@ def parameters(self, vegp_data):
 
         self._ensure_land_covers_set()
 
-        land_cover_items = self._veg_params.select('{LandCoverParamItem}')
+        land_cover_items = self._veg_params.select("{LandCoverParamItem}")
         for key, values in vegp_data.items():
             if key not in ref_dict:
-                raise Exception(f'Unknown vegp key: {key}')
+                raise Exception(f"Unknown vegp key: {key}")
 
             name = ref_dict[key][-1]
             for item, val in zip(land_cover_items, values):
                 item[name] = val
 
                 # Clear the history
-                item._details_[name]['history'] = []
+                item._details_[name]["history"] = []
 
         return self
 
     @with_absolute_path
-    def map_file(self, path='drv_vegm.dat'):
-        vegm_data = read_clm(path, type='vegm')
+    def map_file(self, path="drv_vegm.dat"):
+        vegm_data = read_clm(path, type="vegm")
         self.map(vegm_data)
-        self._import_paths['map'] = path
+        self._import_paths["map"] = path
         return self
 
     def map(self, vegm_data):
         self._ensure_land_covers_set()
 
         land_names = self._land_names
-        all_tokens = [
-            'Latitude',
-            'Longitude',
-            'Sand',
-            'Clay',
-            'Color'
-        ] + land_names
+        all_tokens = ["Latitude", "Longitude", "Sand", "Clay", "Color"] + land_names
 
         # Slice the tokens so they match the shape of the data
         num_tokens = vegm_data.shape[2]
@@ -1368,30 +1498,30 @@ def map(self, vegm_data):
         run_name = self.run.get_name()
 
         for i, token in enumerate(all_tokens):
-            file_name = f'{run_name}_clm_{token.lower()}.pfb'
+            file_name = f"{run_name}_clm_{token.lower()}.pfb"
             if token in land_names:
-                item, = self._veg_map.select(f'LandFrac/{token}')
-                file_name = f'{run_name}_clm_{token}_landfrac.pfb'
+                (item,) = self._veg_map.select(f"LandFrac/{token}")
+                file_name = f"{run_name}_clm_{token}_landfrac.pfb"
             else:
-                item, = self._veg_map.select(token)
+                (item,) = self._veg_map.select(token)
 
             array = vegm_data[:, :, i]
-            if token == 'Color':
+            if token == "Color":
                 # This one needs to be an integer
                 array = array.astype(np.int32)
 
             if array.min() == array.max():
                 # All the values are the same
-                item.Type = 'Constant'
+                item.Type = "Constant"
                 item.Value = array[0, 0].item()
             else:
                 write_pfb(get_absolute_path(file_name), vegm_data[:, :, i])
-                item.Type = 'PFBFile'
+                item.Type = "PFBFile"
                 item.FileName = file_name
 
             # Clear the histories
             for details in item._details_.values():
-                details['history'] = []
+                details["history"] = []
 
         return self
 
@@ -1399,7 +1529,7 @@ def set_default_land_names(self):
         veg_params = self.run.Solver.CLM.Vegetation.Parameters
         veg_params.LandNames = self._default_land_names
         # Erase the history on the land names
-        veg_params.details('LandNames')['history'] = []
+        veg_params.details("LandNames")["history"] = []
         return self
 
     def import_if_needed(self):
@@ -1422,13 +1552,13 @@ def import_if_needed(self):
 
     @property
     def _default_land_names(self):
-        path = 'Solver.CLM.Vegetation.Parameters.LandNames'
-        return self.run.details(path).get('default')
+        path = "Solver.CLM.Vegetation.Parameters.LandNames"
+        return self.run.details(path).get("default")
 
     @property
     def _using_clm(self):
         # We can add other checks here if needed
-        return self.run.Solver.LSM == 'CLM'
+        return self.run.Solver.LSM == "CLM"
 
     @property
     def _driver_data_appears_to_be_set(self):
@@ -1461,18 +1591,21 @@ def _land_names(self):
 
     @property
     def _land_covers_are_set(self):
-        land_param_items = self._veg_params.select('{LandCoverParamItem}')
-        land_map_items = self._veg_map.select('LandFrac/{LandFracCoverMapItem}')
-        return (land_param_items and land_map_items and
-                all(x is not None for x in land_param_items + land_map_items))
+        land_param_items = self._veg_params.select("{LandCoverParamItem}")
+        land_map_items = self._veg_map.select("LandFrac/{LandFracCoverMapItem}")
+        return (
+            land_param_items
+            and land_map_items
+            and all(x is not None for x in land_param_items + land_map_items)
+        )
 
     def _ensure_land_covers_set(self):
         if not self._land_covers_are_set:
-            raise Exception('Land cover items are not set')
+            raise Exception("Land cover items are not set")
 
     @property
     def _import_paths(self):
-        return self.run.__dict__.setdefault('_import_paths_', {})
+        return self.run.__dict__.setdefault("_import_paths_", {})
 
     @staticmethod
     def _clear_all_histories(root):
@@ -1487,8 +1620,8 @@ def recursive_clear_histories(parent):
                     continue
 
                 for details in item._details_.values():
-                    if 'history' in details:
-                        details['history'] = []
+                    if "history" in details:
+                        details["history"] = []
 
                 recursive_clear_histories(item)
 
@@ -1496,13 +1629,13 @@ def recursive_clear_histories(parent):
 
     def _convert_to_domain_type(self, key, value):
         conversion_map = {
-            'IntValue': lambda x: int(float(x)),
-            'DoubleValue': float,
+            "IntValue": lambda x: int(float(x)),
+            "DoubleValue": float,
         }
 
         try:
-            domain_type = 'AnyString'
-            for domain_key in self.run.details(key)['domains']:
+            domain_type = "AnyString"
+            for domain_key in self.run.details(key)["domains"]:
                 if domain_key in conversion_map:
                     domain_type = domain_key
                     break
diff --git a/pftools/python/parflow/tools/compare.py b/pftools/python/parflow/tools/compare.py
new file mode 100644
index 000000000..e268f2c0c
--- /dev/null
+++ b/pftools/python/parflow/tools/compare.py
@@ -0,0 +1,227 @@
+import os
+from parflow.tools.io import read_pfb, read_pfsb
+
+
+def pf_test_equal(a, b, message):
+    pf_eps = 1e-5
+    if abs(a - b) > pf_eps:
+        print(f"FAILED : {message} {a} is not equal to {b}")
+        return False
+
+    return True
+
+
+def msig_diff(data1, data2, m, abs_zero=0.0):
+    """Python version of the C MSigDiff function.
+
+    Two nd arrays are given as the first two arguments.
+    The grid point at which the number of digits in agreement
+    (significant digits) is fewest is determined.  If m >= 0
+    then the coordinate whose two values differ in more than
+    m significant digits will be computed.  If m < 0 then the
+    coordinate whose values have a minimum number of significant
+    digits will be computed. The number of the fewest significant
+    digits is determined, and the maximum absolute difference is
+    computed. The only coordinates that will be considered will be
+    those whose differences are greater than absolute zero.
+
+    Args:
+        data1 (numpy.ndarray): first ndarray
+        data2 (numpy.ndarray): second ndarray
+        m (int): number of significant digits
+        abs_zero (float): threshold below which all numbers are considered to be 0
+
+    Returns:
+        A list of the following form is returned upon success:
+
+               [[i j k s] max_adiff]
+
+        where i, j, and k are the coordinates computed, sd is the
+        minimum number of significant digits computed, and max_adiff
+        is the maximum absolute difference computed.
+
+    Raises:
+        ValueError: If data1 and data2 do not have the same shape.
+    """
+    assert isinstance(abs_zero, float)
+    assert abs_zero >= 0
+
+    if not data1.shape == data2.shape:
+        raise ValueError("Error: Data arrays must have the same dimensions.")
+
+    nx, ny, nz = data1.shape
+
+    if m >= 0:
+        sig_dig_rhs = 0.5 / 10**m
+    else:
+        sig_dig_rhs = 0.0
+
+    m_sig_digs_everywhere = True
+    max_sdiff = 0.0
+    max_adiff = 0.0
+
+    for k in range(nz):
+        for j in range(ny):
+            for i in range(nx):
+                adiff = abs(data1[i, j, k] - data2[i, j, k])
+                amax = max(abs(data1[i, j, k]), abs(data2[i, j, k]))
+
+                if max_adiff < adiff:
+                    max_adiff = adiff
+
+                m_sig_digs = True
+                if amax > abs_zero:
+                    sdiff = adiff / amax
+                    if sdiff > sig_dig_rhs:
+                        m_sig_digs = False
+
+                if not m_sig_digs:
+                    if sdiff > max_sdiff:
+                        max_sdiff = sdiff
+                        mi, mj, mk = i, j, k
+
+                    m_sig_digs_everywhere = False
+
+    result = []
+    if not m_sig_digs_everywhere:
+        sig_digs = 0
+        sdiff = max_sdiff
+        while sdiff <= 0.5e-01:
+            sdiff *= 10.0
+            sig_digs += 1
+
+        result.append([mi, mj, mk, sig_digs])
+        result.append(max_adiff)
+
+    return result
+
+
+def pf_test_file(file, correct_file, message, sig_digits=6):
+    """Python version of the tcl pftestFile procedure.
+
+    Two file paths are given as the first two arguments.
+    The function reads them into ndarrays and calls a comparison
+    function (msig_diff) to check if the files differ by
+    more than sig_digits significant digits in any coordinate.
+    If they do, the function prints an error message and the
+    coordinate in which the files have the greatest difference.
+
+    Args:
+        file (string): path to test output file
+        correct_file (string): path to reference output file
+        message (string): message to print in case of test failure
+        sig_digits (int): number of significant digits
+
+    Returns:
+        bool: True if files differ in no more than sig_digits
+              significant digits in all coordinates. False otherwise.
+
+    Raises:
+        FileNotFoundError: If file or correct_file do not exist.
+        Exception: If the function fails to read either file into an ndarray.
+    """
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"FAILED : output file <{file}> not created")
+
+    if not os.path.exists(correct_file):
+        raise FileNotFoundError(
+            f"FAILED : regression check output file <{correct_file}> does not exist"
+        )
+
+    try:
+        if file.endswith(".pfb"):
+            data = read_pfb(file)
+            correct_data = read_pfb(correct_file)
+        elif file.endswith(".pfsb"):
+            data = read_pfsb(file)
+            correct_data = read_pfsb(correct_file)
+        else:
+            raise ValueError("Unknown parflow file type.")
+    except Exception as e:
+        print("Error: Failed to load data from files...", e)
+        return False
+
+    result = msig_diff(data, correct_data, sig_digits)
+    if (len(result)) == 0:
+        return True
+
+    m_sig_digs, max_abs_diff = result
+
+    i, j, k, sig_digs = m_sig_digs
+
+    print(f"FAILED : {message}")
+    print(f"\tMinimum significant digits at ({i:3d}, {j:3d}, {k:3d}) = {sig_digs:2d}")
+    print(f"\tCorrect value {correct_data[i, j, k]:e}")
+    print(f"\tComputed value {data[i, j, k]:e}")
+
+    elt_diff = abs(data[i, j, k] - correct_data[i, j, k])
+    print(f"\tDifference {elt_diff:e}")
+
+    print(f"\tMaximum absolute difference = {max_abs_diff:e}")
+
+    return False
+
+
+def pf_test_file_with_abs(file, correct_file, message, abs_value, sig_digits=6):
+    """Python version of the tcl pftestFileWithAbs procedure.
+
+    Two file paths are given as the first two arguments.
+    The function reads them into ndarrays and calls a comparison
+    function (msig_diff) to check if the files differ by
+    more than sig_digits significant digits in any coordinate.
+    If they do, the function checks if the difference in that
+    coordinate is greater than abs_value. If it is, it prints
+    an error message and the coordinate in which the files have
+    the greatest difference.
+
+    Args:
+        file (string): path to test output file
+        correct_file (string): path to reference output file
+        message (string): message to print in case of test failure
+        abs_value (float): threshold to determine if two files differ
+                           enough to fail the test
+        sig_digits (int): number of significant digits
+
+    Returns:
+        bool: True if files differ in no more than sig_digits
+              significant digits in all coordinates. False otherwise.
+
+    Raises:
+        FileNotFoundError: If file or correct_file do not exist.
+        Exception: If the function fails to read either file into an ndarray.
+    """
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"FAILED : output file <{file}> not created")
+
+    if not os.path.exists(correct_file):
+        raise FileNotFoundError(
+            f"FAILED : regression check output file <{correct_file}> does not exist"
+        )
+
+    try:
+        data = read_pfb(file)
+        correct_data = read_pfb(correct_file)
+    except Exception as e:
+        print("Error: Failed to load data from files.", e)
+        return False
+
+    result = msig_diff(data, correct_data, sig_digits)
+    if len(result) != 0:
+        m_sig_digs, max_abs_diff = result
+
+        i, j, k, sig_digs = m_sig_digs
+
+        elt_diff = abs(data[i, j, k] - correct_data[i, j, k])
+
+        if elt_diff > abs_value:
+            print(f"FAILED : {message}")
+            print(
+                f"\tMinimum significant digits at ({i:3d}, {j:3d}, {k:3d}) = {sig_digs:2d}"
+            )
+            print(f"\tCorrect value {correct_data[i, j, k]:e}")
+            print(f"\tComputed value {data[i, j, k]:e}")
+            print(f"\tDifference {elt_diff:e}")
+            print(f"\tMaximum absolute difference = {max_abs_diff:e}")
+            return False
+
+    return True
diff --git a/pftools/python/parflow/tools/core.py b/pftools/python/parflow/tools/core.py
index 360ba0e70..dfd7556dd 100644
--- a/pftools/python/parflow/tools/core.py
+++ b/pftools/python/parflow/tools/core.py
@@ -13,6 +13,8 @@
 import sys
 import argparse
 
+import numpy as np
+
 from . import settings
 from .fs import get_absolute_path
 from .io import (
@@ -21,9 +23,8 @@
     write_dict,
     read_pfb,
     write_pfb,
-    write_dist
+    write_dist,
 )
-from .terminal import Symbols as TermSymbol
 
 from .database.generated import BaseRun
 from .export import CLMExporter, SubsurfacePropertiesExporter
@@ -43,25 +44,22 @@ def check_parflow_execution(out_file):
     print(f'# {"=" * 78}')
     execute_success = False
     if Path(out_file).exists():
-        with open(out_file, 'r') as f:
+        with open(out_file, "r") as f:
             contents = f.read()
-            if 'Problem solved' in contents:
-                emoji = f'{TermSymbol.splash} '
-                print(
-                    f'# ParFlow ran successfully {emoji * 3}')
+            if "Problem solved" in contents:
+                print(f"# ParFlow ran successfully")
                 execute_success = True
             else:
-                emoji = f'{TermSymbol.x} '
-                print(f'# ParFlow run failed. {emoji * 3} '
-                      f'Contents of error output file:')
+                print(f"# ParFlow run failed. " f"Contents of error output file:")
                 print("-" * 80)
                 print(contents)
                 print("-" * 80)
     else:
-        print(f'# Cannot find {out_file} in {os.getcwd()}')
+        print(f"# Cannot find {out_file} in {os.getcwd()}")
     print(f'# {"=" * 78}')
     return execute_success
 
+
 # -----------------------------------------------------------------------------
 
 
@@ -78,20 +76,22 @@ def get_current_parflow_version():
         str: Return ParFlow version like '3.6.0'
 
     """
-    version = '3.6.0'
+    version = "3.6.0"
     version_file = f'{os.getenv("PARFLOW_DIR")}/config/pf-cmake-env.sh'
     if Path(version_file).resolve().exists():
-        with open(version_file, 'r') as f:
+        with open(version_file, "r") as f:
             for line in f:
-                if 'PARFLOW_VERSION=' in line:
+                if "PARFLOW_VERSION=" in line:
                     version = line[17:-2]
             if not version:
-                print(f'Cannot find version in {version_file}')
+                print(f"Cannot find version in {version_file}")
     else:
-        print(f'Cannot find environment file in '
-              f'{str(Path(version_file).resolve())}.')
+        print(
+            f"Cannot find environment file in " f"{str(Path(version_file).resolve())}."
+        )
     return version
 
+
 # -----------------------------------------------------------------------------
 
 
@@ -102,85 +102,114 @@ def get_process_args():
     parser = argparse.ArgumentParser(description="Parflow run arguments")
 
     # ++++++++++++++++
-    group = parser.add_argument_group('Parflow settings')
-    group.add_argument("--parflow-directory",
-                       default=None,
-                       dest="parflow_directory",
-                       help="Path to use for PARFLOW_DIR")
-    group.add_argument("--parflow-version",
-                       default=None,
-                       dest="parflow_version",
-                       help="Override detected Parflow version")
+    group = parser.add_argument_group("Parflow settings")
+    group.add_argument(
+        "--parflow-directory",
+        default=None,
+        dest="parflow_directory",
+        help="Path to use for PARFLOW_DIR",
+    )
+    group.add_argument(
+        "--parflow-version",
+        default=None,
+        dest="parflow_version",
+        help="Override detected Parflow version",
+    )
     # ++++++++++++++++
-    group = parser.add_argument_group('Execution settings')
-    group.add_argument("--working-directory",
-                       default=None,
-                       dest="working_directory",
-                       help="Path to execution working directory")
-
-    group.add_argument("--skip-validation",
-                       default=False,
-                       dest="skip_validation",
-                       action='store_true',
-                       help="Disable validation pass")
-
-    group.add_argument("--dry-run",
-                       default=False,
-                       action='store_true',
-                       dest="dry_run",
-                       help="Prevent execution")
+    group = parser.add_argument_group("Execution settings")
+    group.add_argument(
+        "--working-directory",
+        default=None,
+        dest="working_directory",
+        help="Path to execution working directory",
+    )
+
+    group.add_argument(
+        "--skip-validation",
+        default=False,
+        dest="skip_validation",
+        action="store_true",
+        help="Disable validation pass",
+    )
+
+    group.add_argument(
+        "--dry-run",
+        default=False,
+        action="store_true",
+        dest="dry_run",
+        help="Prevent execution",
+    )
     # ++++++++++++++++
-    group = parser.add_argument_group('Error handling settings')
-    group.add_argument("--show-line-error",
-                       default=False,
-                       dest="show_line_error",
-                       action='store_true',
-                       help="Show line error")
-
-    group.add_argument("--exit-on-error",
-                       default=False,
-                       dest="exit_on_error",
-                       action='store_true',
-                       help="Exit at error")
+    group = parser.add_argument_group("Error handling settings")
+    group.add_argument(
+        "--show-line-error",
+        default=False,
+        dest="show_line_error",
+        action="store_true",
+        help="Show line error",
+    )
+
+    group.add_argument(
+        "--exit-on-error",
+        default=False,
+        dest="exit_on_error",
+        action="store_true",
+        help="Exit at error",
+    )
     # ++++++++++++++++
-    group = parser.add_argument_group('Additional output')
-    group.add_argument("--write-yaml",
-                       default=False,
-                       dest="write_yaml",
-                       action='store_true',
-                       help="Enable config to be written as YAML file")
-
-    group.add_argument("--validation-verbose",
-                       default=False,
-                       dest="validation_verbose",
-                       action='store_true',
-                       help="Only print validation results for "
-                            "key/value pairs with errors")
-
-    group.add_argument("--overwrite-clm-driver-files",
-                       default=False,
-                       dest="overwrite_clm_driver_files",
-                       action='store_true',
-                       help="Allow old clm driver files to be overwritten")
+    group = parser.add_argument_group("Additional output")
+    group.add_argument(
+        "--write-yaml",
+        default=False,
+        dest="write_yaml",
+        action="store_true",
+        help="Enable config to be written as YAML file",
+    )
+
+    group.add_argument(
+        "--validation-verbose",
+        default=False,
+        dest="validation_verbose",
+        action="store_true",
+        help="Only print validation results for " "key/value pairs with errors",
+    )
+
+    group.add_argument(
+        "--overwrite-clm-driver-files",
+        default=False,
+        dest="overwrite_clm_driver_files",
+        action="store_true",
+        help="Allow old clm driver files to be overwritten",
+    )
 
     # ++++++++++++++++
-    group = parser.add_argument_group('Parallel execution')
-    group.add_argument("-p", type=int, default=0,
-                       dest="p",
-                       help="P allocates the number of processes "
-                            "to the grid-cells in x")
-    group.add_argument("-q", type=int, default=0,
-                       dest="q",
-                       help="Q allocates the number of processes "
-                            "to the grid-cells in y")
-    group.add_argument("-r", type=int, default=0,
-                       dest="r",
-                       help="R allocates the number of processes "
-                            "to the grid-cells in z")
+    group = parser.add_argument_group("Parallel execution")
+    group.add_argument(
+        "-p",
+        type=int,
+        default=0,
+        dest="p",
+        help="P allocates the number of processes " "to the grid-cells in x",
+    )
+    group.add_argument(
+        "-q",
+        type=int,
+        default=0,
+        dest="q",
+        help="Q allocates the number of processes " "to the grid-cells in y",
+    )
+    group.add_argument(
+        "-r",
+        type=int,
+        default=0,
+        dest="r",
+        help="R allocates the number of processes " "to the grid-cells in z",
+    )
 
     args, unknown = parser.parse_known_args()
     return args
 
+
 # -----------------------------------------------------------------------------
 
 
@@ -216,14 +245,15 @@ def update_run_from_args(run, args):
     if args.overwrite_clm_driver_files:
         clm_solver = run.Solver.CLM
         keys = [
-            'OverwriteDrvClmin',
-            'OverwriteDrvVegp',
-            'OverwriteDrvVegm',
+            "OverwriteDrvClmin",
+            "OverwriteDrvVegp",
+            "OverwriteDrvVegm",
         ]
         for key in keys:
             if getattr(clm_solver, key) is False:
                 setattr(clm_solver, key, True)
 
+
 # -----------------------------------------------------------------------------
 
 
@@ -239,6 +269,7 @@ class Run(BaseRun):
             directory where the python executable was run from.
 
     """
+
     def __init__(self, name, basescript=None):
         super().__init__(None)
         self._accessor_ = None
@@ -272,50 +303,48 @@ def from_definition(cls, file_path):
         file_path = Path(get_absolute_path(file_path))
         name, ext = file_path.stem, file_path.suffix[1:]
 
-        ext_map = {
-            'yaml': 'yaml_file',
-            'yml': 'yaml_file',
-            'pfidb': 'pfidb_file'
-        }
+        ext_map = {"yaml": "yaml_file", "yml": "yaml_file", "pfidb": "pfidb_file"}
 
         if ext not in ext_map:
-            raise Exception(f'Unknown extension: {ext}')
+            raise Exception(f"Unknown extension: {ext}")
 
         new_run = cls(name, file_path)
         kwargs = {ext_map[ext]: file_path}
         new_run.pfset(silence_if_undefined=True, **kwargs)
 
         # Try to solve order sensitive property settings
-        while '_pfstore_' in new_run.__dict__:
-            invalid_props = new_run.__dict__.pop('_pfstore_')
+        while "_pfstore_" in new_run.__dict__:
+            invalid_props = new_run.__dict__.pop("_pfstore_")
             previous_size = len(invalid_props)
             for key, value in invalid_props.items():
                 new_run.pfset(key, value, silence_if_undefined=True)
 
             # Break if no key was able to be mapped outside pfstore
-            if ('_pfstore_' in new_run.__dict__ and
-                    previous_size == len(new_run.__dict__['_pfstore_'])):
+            if "_pfstore_" in new_run.__dict__ and previous_size == len(
+                new_run.__dict__["_pfstore_"]
+            ):
                 break
 
         # Print any remaining key with no mapping
-        if '_pfstore_' in new_run.__dict__:
-            invalid_props = new_run.__dict__.pop('_pfstore_')
+        if "_pfstore_" in new_run.__dict__:
+            invalid_props = new_run.__dict__.pop("_pfstore_")
             for key, value in invalid_props.items():
                 new_run.pfset(key, value)
 
-        if ext == 'pfidb':
+        if ext == "pfidb":
             # Import CLM files if we need to
             try:
                 CLMImporter(new_run).import_if_needed()
             except Exception:
-                print(' => Error during CLM import - '
-                      'CLM specific key have been skipped')
+                print(
+                    " => Error during CLM import - "
+                    "CLM specific key have been skipped"
+                )
 
         return new_run
 
     def get_name(self):
-        """Returns name of run
-        """
+        """Returns name of run"""
         return self._name_
 
     def set_name(self, new_name):
@@ -335,8 +364,7 @@ def data_accessor(self):
             self._accessor_ = DataAccessor(self)
         return self._accessor_
 
-    def write(self, file_name=None, file_format='pfidb',
-              working_directory=None):
+    def write(self, file_name=None, file_format="pfidb", working_directory=None):
         """Method to write database file to disk
 
         Args:
@@ -357,11 +385,13 @@ def write(self, file_name=None, file_format='pfidb',
         if working_directory:
             settings.set_working_directory(working_directory)
 
-        f_name = os.path.join(settings.WORKING_DIRECTORY,
-                              f'{self._name_}.{file_format}')
+        f_name = os.path.join(
+            settings.WORKING_DIRECTORY, f"{self._name_}.{file_format}"
+        )
         if file_name:
-            f_name = os.path.join(settings.WORKING_DIRECTORY,
-                                  f'{file_name}.{file_format}')
+            f_name = os.path.join(
+                settings.WORKING_DIRECTORY, f"{file_name}.{file_format}"
+            )
         full_file_path = os.path.abspath(f_name)
         write_dict(self.to_dict(), full_file_path)
 
@@ -372,7 +402,7 @@ def write(self, file_name=None, file_format='pfidb',
         # revert working directory to original directory
         settings.set_working_directory(prev_dir)
 
-        return full_file_path, full_file_path[:-(len(file_format)+1)]
+        return full_file_path, full_file_path[: -(len(file_format) + 1)]
 
     def write_subsurface_table(self, file_name=None, working_directory=None):
         # overwrite current working directory
@@ -381,10 +411,10 @@ def write_subsurface_table(self, file_name=None, working_directory=None):
             settings.set_working_directory(working_directory)
 
         if file_name is None:
-            file_name = f'{self._name_}_subsurface.csv'
+            file_name = f"{self._name_}_subsurface.csv"
         full_path = get_absolute_path(file_name)
         exporter = SubsurfacePropertiesExporter(self)
-        if file_name.lower().endswith('.csv'):
+        if file_name.lower().endswith(".csv"):
             exporter.write_csv(full_path)
         else:
             exporter.write_txt(full_path)
@@ -439,19 +469,19 @@ def run(self, working_directory=None, skip_validation=False):
 
         print()
         print(f'# {"=" * 78}')
-        print('# ParFlow directory')
+        print("# ParFlow directory")
         print(f'#  - {os.getenv("PARFLOW_DIR")}')
-        print('# ParFlow version')
-        print(f'#  - {settings.PARFLOW_VERSION}')
-        print('# Working directory')
-        print(f'#  - {os.path.dirname(file_name)}')
-        print('# ParFlow database')
-        print(f'#  - {os.path.basename(file_name)}')
+        print("# ParFlow version")
+        print(f"#  - {settings.PARFLOW_VERSION}")
+        print("# Working directory")
+        print(f"#  - {os.path.dirname(file_name)}")
+        print("# ParFlow database")
+        print(f"#  - {os.path.basename(file_name)}")
         print(f'# {"=" * 78}')
 
         # Only write YAML in run()
         if self._process_args_.write_yaml:
-            full_path, no_extension = self.write(file_format='yaml')
+            full_path, no_extension = self.write(file_format="yaml")
             print(f'YAML output: "{full_path}"')
 
         print()
@@ -471,8 +501,8 @@ def run(self, working_directory=None, skip_validation=False):
             prev_dir = os.getcwd()
             try:
                 os.chdir(settings.WORKING_DIRECTORY)
-                os.system(f'sh $PARFLOW_DIR/bin/run {run_file} {num_procs}')
-                success = check_parflow_execution(f'{run_file}.out.txt')
+                os.system(f"sh $PARFLOW_DIR/bin/run {run_file} {num_procs}")
+                success = check_parflow_execution(f"{run_file}.out.txt")
             finally:
                 os.chdir(prev_dir)
 
@@ -484,6 +514,39 @@ def run(self, working_directory=None, skip_validation=False):
         if not success or error_count > 0:
             sys.exit(1)
 
+    def check_nans(self, working_directory, include_forcing=True):
+        """Check the input files for NaNs.
+
+        Args:
+            working_directory: The working directory of the ParFlow run.
+            include_forcing (bool): If set to True, forcing files are going
+                to be checked.
+
+        Raises:
+            ValueError: If an input file contains a NaN.
+        """
+
+        runscript_path, _ = self.write(file_format="yaml")
+        all_files = []
+        with open(runscript_path, "r") as f:
+            for line in f:
+                line = line.strip()
+                if line.startswith("FileName: ") and line.endswith(".pfb"):
+                    path = os.path.join(working_directory, line.split()[1])
+                    all_files.append(path)
+
+        if include_forcing:
+            run = self.__class__.from_definition(runscript_path)
+            forcing_dir = run.Solver.CLM.MetFilePath
+            for file in os.listdir(forcing_dir):
+                if file.endswith(".pfb"):
+                    all_files.append(os.path.join(forcing_dir, file))
+
+        for file in all_files:
+            data = read_pfb(file)
+            if np.any(np.isnan(data)):
+                raise ValueError(f"{file} contains NaN values.")
+
     def dist(self, pfb_file, **kwargs):
         """Distribute a PFB file using the P/Q/R settings from the run
         or override them with the provided arguments.
@@ -492,21 +555,20 @@ def dist(self, pfb_file, **kwargs):
           - NX, NY, NZ...
         """
         # Any provided args should override the scripts ones
+
         update_run_from_args(self, self._process_args_)
 
         pfb_file_full_path = get_absolute_path(pfb_file)
-        p = kwargs.get('P', self.Process.Topology.P)
-        q = kwargs.get('Q', self.Process.Topology.Q)
-        r = kwargs.get('R', self.Process.Topology.R)
+        p = kwargs.get("P", self.Process.Topology.P)
+        q = kwargs.get("Q", self.Process.Topology.Q)
+        r = kwargs.get("R", self.Process.Topology.R)
 
-        with ParflowBinaryReader(pfb_file_full_path) as pfb:
+        with ParflowBinaryReader(pfb_file_full_path, read_sg_info=True) as pfb:
             array = pfb.read_all_subgrids()
             header = pfb.header
 
-        dx, dy, dz = header['dx'], header['dy'], header['dz']
-        write_pfb(pfb_file_full_path, array,
-                   p=p, q=q, r=r, dx=dx, dy=dy, dz=dz,
-                   dist=True)
-
-
+        dx, dy, dz = header["dx"], header["dy"], header["dz"]
 
+        write_pfb(
+            pfb_file_full_path, array, p=p, q=q, r=r, dx=dx, dy=dy, dz=dz, dist=True
+        )
diff --git a/pftools/python/parflow/tools/database/core.py b/pftools/python/parflow/tools/database/core.py
index c30f7ab45..4c19d2ebe 100644
--- a/pftools/python/parflow/tools/database/core.py
+++ b/pftools/python/parflow/tools/database/core.py
@@ -2,14 +2,19 @@
 This module aims to provide the core components that are required to build
 a Parflow input deck.
 """
+
 import sys
 import yaml
 
 from parflow.tools import settings
 from parflow.tools.fs import get_text_file_content
 from parflow.tools.helper import (
-    map_to_child, map_to_children_of_type, map_to_parent, map_to_self,
-    remove_prefix, filter_none
+    map_to_child,
+    map_to_children_of_type,
+    map_to_parent,
+    map_to_self,
+    remove_prefix,
+    filter_none,
 )
 from parflow.tools.helper import normalize_location, sort_dict_by_priority
 from parflow.tools.io import read_pfidb
@@ -22,37 +27,45 @@
 # Accessor helpers
 # ----------------------------------------------------------------------------
 
+
 def validate_helper(container, name, value, indent):
-    """Helper function for validating a value
-    """
+    """Helper function for validating a value"""
     num_errors = 0
-    validation_string = ''
+    validation_string = ""
     details = container._details_[name]
-    has_default = 'default' in details
-    history = details.get('history')
-    if (has_default and value == details['default'] and
-            'MandatoryValue' not in details['domains']):
+    has_default = "default" in details
+    history = details.get("history")
+    if (
+        has_default
+        and value == details["default"]
+        and "MandatoryValue" not in details["domains"]
+    ):
         pass
     else:
-        num_errors, validation_string = \
-            validate_value_to_string(
-                container, value, has_default, details['domains'],
-                container.get_context_settings(), history, indent)
+        num_errors, validation_string = validate_value_to_string(
+            container,
+            value,
+            has_default,
+            details["domains"],
+            container.get_context_settings(),
+            history,
+            indent,
+        )
 
     return num_errors, validation_string
 
+
 # -----------------------------------------------------------------------------
 
 
 def detail_helper(container, name, value):
-    """Helper function that extract elements of the field's detail
-    """
+    """Helper function that extract elements of the field's detail"""
     details = container._details_.get(name, {})
-    domains = details.get('domains')
-    handlers = details.get('handlers')
-    crosscheck = details.get('crosscheck')
+    domains = details.get("domains")
+    handlers = details.get("handlers")
+    crosscheck = details.get("crosscheck")
     if details:
-        history = details.setdefault('history', [])
+        history = details.setdefault("history", [])
         history.append(value)
     else:
         history = None
@@ -64,16 +77,18 @@ def detail_helper(container, name, value):
 # Internal field name helpers
 # -----------------------------------------------------------------------------
 
+
 def is_private_key(name):
     """Test if the given key is a key or a private member
 
     Return True if it is a private member
     """
-    return name[0] == '_' and name[-1] == '_'
+    return name[0] == "_" and name[-1] == "_"
 
 
 # -----------------------------------------------------------------------------
 
+
 def is_not_private_key(name):
     """Test if the given key is a key or a private member
 
@@ -84,25 +99,26 @@ def is_not_private_key(name):
 
 # -----------------------------------------------------------------------------
 
+
 def to_str_dict_format(value):
-    """Ensure that the output value is a valid string
-    """
+    """Ensure that the output value is a valid string"""
     if isinstance(value, str):
         return value
 
-    if hasattr(value, '__iter__'):
-        return ' '.join([str(v) for v in value])
+    if hasattr(value, "__iter__"):
+        return " ".join([str(v) for v in value])
 
     return value
 
 
 # -----------------------------------------------------------------------------
 
-def extract_keys_from_object(dict_to_fill, instance, parent_namespace=''):
+
+def extract_keys_from_object(dict_to_fill, instance, parent_namespace=""):
     """Method that walk PFDBObj object and record their key and value
     inside a Python dict.
     """
-    if hasattr(instance, '_pfstore_'):
+    if hasattr(instance, "_pfstore_"):
         for key, value in instance._pfstore_.items():
             dict_to_fill[key] = to_str_dict_format(value)
 
@@ -113,20 +129,18 @@ def extract_keys_from_object(dict_to_fill, instance, parent_namespace=''):
 
         full_qualified_key = instance.to_pf_name(parent_namespace, key)
         if isinstance(value, PFDBObj):
-            if hasattr(value, '_value_'):
-                has_details = hasattr(value, '_details_') \
-                    and '_value_' in value._details_
-                details = value._details_['_value_'] if has_details else None
-                has_default = has_details and 'default' in details
-                has_domain = has_details and 'domains' in details
-                is_mandatory = has_domain \
-                    and 'MandatoryValue' in details['domains']
-                is_default = has_default and \
-                    value._value_ == details['default']
-                is_set = has_details and details.get('history')
+            if hasattr(value, "_value_"):
+                has_details = (
+                    hasattr(value, "_details_") and "_value_" in value._details_
+                )
+                details = value._details_["_value_"] if has_details else None
+                has_default = has_details and "default" in details
+                has_domain = has_details and "domains" in details
+                is_mandatory = has_domain and "MandatoryValue" in details["domains"]
+                is_default = has_default and value._value_ == details["default"]
+                is_set = has_details and details.get("history")
                 if is_mandatory or not is_default or is_set:
-                    dict_to_fill[full_qualified_key] = \
-                        to_str_dict_format(value._value_)
+                    dict_to_fill[full_qualified_key] = to_str_dict_format(value._value_)
             extract_keys_from_object(dict_to_fill, value, full_qualified_key)
         else:
             dict_to_fill[full_qualified_key] = to_str_dict_format(value)
@@ -134,26 +148,24 @@ def extract_keys_from_object(dict_to_fill, instance, parent_namespace=''):
 
 # -----------------------------------------------------------------------------
 
-def extract_keys_from_dict(dict_to_fill, dict_obj, parent_namespace=''):
+
+def extract_keys_from_dict(dict_to_fill, dict_obj, parent_namespace=""):
     """Helper function to extract a flat key/value dictionary for
     a given PFDBObj inside dict_to_fill.
     """
     for key, value in dict_obj.items():
-        if parent_namespace and key == '_value_':
+        if parent_namespace and key == "_value_":
             dict_to_fill[parent_namespace] = value
             continue
 
         if value is None or is_private_key(key):
             continue
 
-        full_qualified_key = (
-            f'{parent_namespace}.{key}' if parent_namespace else key
-        )
+        full_qualified_key = f"{parent_namespace}.{key}" if parent_namespace else key
         if isinstance(value, dict):
             # Need to handle _value_
-            if hasattr(value, '_value_'):
-                dict_to_fill[full_qualified_key] = to_str_dict_format(
-                    value._value_)
+            if hasattr(value, "_value_"):
+                dict_to_fill[full_qualified_key] = to_str_dict_format(value._value_)
             extract_keys_from_dict(dict_to_fill, value, full_qualified_key)
         else:
             dict_to_fill[full_qualified_key] = to_str_dict_format(value)
@@ -161,12 +173,13 @@ def extract_keys_from_dict(dict_to_fill, dict_obj, parent_namespace=''):
 
 # -----------------------------------------------------------------------------
 
+
 def flatten_hierarchical_map(hierarchical_map):
     """Helper function that take a hierarchical map and return a flat
     version of it.
     """
     flat_map = {}
-    extract_keys_from_dict(flat_map, hierarchical_map, parent_namespace='')
+    extract_keys_from_dict(flat_map, hierarchical_map, parent_namespace="")
     return flat_map
 
 
@@ -174,9 +187,10 @@ def flatten_hierarchical_map(hierarchical_map):
 # Main DB Object
 # -----------------------------------------------------------------------------
 
+
 class PFDBObj:
-    """Core ParFlow Database Object node
-    """
+    """Core ParFlow Database Object node"""
+
     def __init__(self, parent=None):
         """
         Create container object while keeping a reference to your parent
@@ -200,32 +214,35 @@ def __setattr__(self, name, value):
         handlers = None
         history = None
         value_object_assignment = False
-        if is_not_private_key(name) and hasattr(self, '_details_'):
+        if is_not_private_key(name) and hasattr(self, "_details_"):
             if name in self._details_:
                 domains, handlers, history, crosscheck = detail_helper(
-                    self, name, value)
-            elif hasattr(self, name) and isinstance(self.__dict__[name],
-                                                    PFDBObj):
+                    self, name, value
+                )
+            elif hasattr(self, name) and isinstance(self.__dict__[name], PFDBObj):
                 # Handle value object assignment
                 value_object_assignment = True
                 value_obj = self.__dict__[name]
                 domains, handlers, history, crosscheck = detail_helper(
-                    value_obj, '_value_', value)
+                    value_obj, "_value_", value
+                )
             else:
-                msg = (f'{self.full_name()}: Field {name} is not part of the '
-                       f'expected schema {self.__class__}')
+                msg = (
+                    f"{self.full_name()}: Field {name} is not part of the "
+                    f"expected schema {self.__class__}"
+                )
                 print(msg)
                 if settings.EXIT_ON_ERROR:
                     raise ValueError(msg)
 
         # Run domain validation
         if settings.PRINT_LINE_ERROR:
-            validate_value_with_exception(
-                value, domains, settings.EXIT_ON_ERROR)
+            validate_value_with_exception(value, domains, settings.EXIT_ON_ERROR)
 
         if value_object_assignment:
-            self.__dict__[name].__dict__['_value_'] = \
-                decorate_value(value, self, handlers)
+            self.__dict__[name].__dict__["_value_"] = decorate_value(
+                value, self, handlers
+            )
         else:
             # Decorate value if need be (i.e. Geom.names: 'a b c')
             self.__dict__[name] = decorate_value(value, self, handlers)
@@ -241,7 +258,7 @@ def __len__(self):
         """
         value_count = 0
 
-        if hasattr(self, '_value_') and self._value_ is not None:
+        if hasattr(self, "_value_") and self._value_ is not None:
             value_count += 1
 
         for name in self.keys(True):
@@ -250,9 +267,12 @@ def __len__(self):
                 value_count += len(obj)
             elif obj is not None:
                 value_count += 1
-            elif hasattr(self, '_details_') and name in self._details_ \
-                    and 'domains' in self._details_[name]:
-                if 'MandatoryValue' in self._details_[name]['domains']:
+            elif (
+                hasattr(self, "_details_")
+                and name in self._details_
+                and "domains" in self._details_[name]
+            ):
+                if "MandatoryValue" in self._details_[name]["domains"]:
                     value_count += 1
 
         return value_count
@@ -271,14 +291,16 @@ def __getitem__(self, key):
         if hasattr(self, key_str):
             return getattr(self, key_str, None)
 
-        prefix = ''
-        if hasattr(self, '_details_') and '_prefix_' in self._details_:
-            prefix = self._details_['_prefix_']
+        prefix = ""
+        if hasattr(self, "_details_") and "_prefix_" in self._details_:
+            prefix = self._details_["_prefix_"]
 
-        key_str = f'{prefix}{key_str}'
+        key_str = f"{prefix}{key_str}"
         if not hasattr(self, key_str):
-            print(f'Could not find key {key}/{key_str} in '
-                  f'{list(self.__dict__.keys())}')
+            print(
+                f"Could not find key {key}/{key_str} in "
+                f"{list(self.__dict__.keys())}"
+            )
 
         return getattr(self, key_str, None)
 
@@ -303,7 +325,7 @@ def keys(self, skip_default=False):
         """
         for name in self.__dict__:
             if name is None:
-                print('need to fix the children instantiator')
+                print("need to fix the children instantiator")
                 continue
 
             if is_private_key(name):
@@ -315,16 +337,14 @@ def keys(self, skip_default=False):
                     yield name
 
             else:
-                has_details = hasattr(self, '_details_') \
-                    and name in self._details_
+                has_details = hasattr(self, "_details_") and name in self._details_
                 details = self._details_[name] if has_details else None
-                has_default = has_details and 'default' in details
-                has_domain = has_details and 'domains' in details
-                is_ignored = has_details and 'ignore' in details
-                is_mandatory = has_domain \
-                    and 'MandatoryValue' in details['domains']
-                is_default = has_default and obj == details['default']
-                is_set = has_details and details.get('history')
+                has_default = has_details and "default" in details
+                has_domain = has_details and "domains" in details
+                is_ignored = has_details and "ignore" in details
+                is_mandatory = has_domain and "MandatoryValue" in details["domains"]
+                is_default = has_default and obj == details["default"]
+                is_set = has_details and details.get("history")
 
                 if is_ignored:
                     continue
@@ -341,8 +361,9 @@ def keys(self, skip_default=False):
 
     # -------------------------------------------------------------------------
 
-    def validate(self, indent=1, verbose=False, enable_print=True,
-                 working_directory=None):
+    def validate(
+        self, indent=1, verbose=False, enable_print=True, working_directory=None
+    ):
         """
         Method to validate sub hierarchy
         """
@@ -355,37 +376,37 @@ def validate(self, indent=1, verbose=False, enable_print=True,
             settings.set_working_directory(working_directory)
 
         error_count = 0
-        indent_str = '  '*indent
+        indent_str = "  " * indent
         for name in self.keys(skip_default=True):
             obj = self.__dict__[name]
             if isinstance(obj, PFDBObj):
                 if len(obj):
-                    if hasattr(obj, '_value_'):
+                    if hasattr(obj, "_value_"):
                         value = obj._value_
                         add_errors, validation_string = validate_helper(
-                            obj, '_value_', value, indent)
+                            obj, "_value_", value, indent
+                        )
 
                         if enable_print and (add_errors or verbose):
-                            print(f'{indent_str}{name}: {validation_string}')
+                            print(f"{indent_str}{name}: {validation_string}")
 
                         error_count += add_errors
                     elif enable_print:
                         if verbose or obj.validate(enable_print=False):
-                            print(f'{indent_str}{name}:')
+                            print(f"{indent_str}{name}:")
 
-                    error_count += obj.validate(indent + 1,
-                                                verbose=verbose,
-                                                enable_print=enable_print)
+                    error_count += obj.validate(
+                        indent + 1, verbose=verbose, enable_print=enable_print
+                    )
 
-            elif hasattr(self, '_details_') and name in self._details_:
-                add_errors, validation_string = validate_helper(
-                    self, name, obj, indent)
+            elif hasattr(self, "_details_") and name in self._details_:
+                add_errors, validation_string = validate_helper(self, name, obj, indent)
                 if enable_print and (verbose or add_errors):
-                    print(f'{indent_str}{name}: {validation_string}')
+                    print(f"{indent_str}{name}: {validation_string}")
                 error_count += add_errors
             elif obj is not None:
                 if enable_print and verbose:
-                    print(f'{indent_str}{name}: {obj}')
+                    print(f"{indent_str}{name}: {obj}")
 
         # revert working directory to original directory
         settings.set_working_directory(prev_dir)
@@ -414,7 +435,7 @@ def full_name(self):
                 return f'not found {count}: {".".join(full_path)}'
 
         full_path.reverse()
-        return '.'.join(full_path)
+        return ".".join(full_path)
 
     # -------------------------------------------------------------------------
 
@@ -425,16 +446,16 @@ def to_pf_name(self, parent_namespace, key):
         can be defined in Python vs Parflow key.
         """
         value = self.__dict__[key]
-        prefix = ''
+        prefix = ""
         if isinstance(value, PFDBObj):
             if value._prefix_ and key.startswith(value._prefix_):
                 prefix = value._prefix_
         elif key in self._details_:
             detail = self._details_[key]
-            if '_prefix_' in detail:
-                prefix = detail['_prefix_']
+            if "_prefix_" in detail:
+                prefix = detail["_prefix_"]
 
-        start = f'{parent_namespace}.' if parent_namespace else ''
+        start = f"{parent_namespace}." if parent_namespace else ""
         return start + remove_prefix(key, prefix)
 
     # -------------------------------------------------------------------------
@@ -444,7 +465,7 @@ def get_children_of_type(self, class_name):
         our children.
         """
         results = []
-        for (key, value) in self.__dict__.items():
+        for key, value in self.__dict__.items():
             if is_private_key(key):
                 continue
             if value.__class__.__name__ == class_name:
@@ -455,7 +476,7 @@ def get_children_of_type(self, class_name):
     # -------------------------------------------------------------------------
 
     @normalize_location
-    def select(self, location='.'):
+    def select(self, location="."):
         """
         Return a PFDBObj object based on a location.
 
@@ -466,8 +487,8 @@ def select(self, location='.'):
           run.Process.Topology.select('/Geom') => run.Geom
         """
         current_location = self
-        path_items = location.split('/')
-        if location.startswith('/'):
+        path_items = location.split("/")
+        if location.startswith("/"):
             while current_location._parent_ is not None:
                 current_location = current_location._parent_
 
@@ -479,16 +500,17 @@ def select(self, location='.'):
             current_list = next_list
             next_list = []
 
-            if path_item == '..':
+            if path_item == "..":
                 next_list.extend(map(map_to_parent, current_list))
-            elif path_item == '.':
+            elif path_item == ".":
                 next_list.extend(map(map_to_self, current_list))
-            elif path_item.startswith('{'):
-                multi_list = map(map_to_children_of_type(
-                    path_item[1:-1]), current_list)
+            elif path_item.startswith("{"):
+                multi_list = map(map_to_children_of_type(path_item[1:-1]), current_list)
                 next_list = [x for sublist in multi_list for x in sublist]
             else:
-                next_list.extend(filter(filter_none, map(map_to_child(path_item), current_list)))
+                next_list.extend(
+                    filter(filter_none, map(map_to_child(path_item), current_list))
+                )
                 if next_list and isinstance(next_list[0], list):
                     next_list = [x for sublist in next_list for x in sublist]
 
@@ -497,7 +519,7 @@ def select(self, location='.'):
     # -------------------------------------------------------------------------
 
     @normalize_location
-    def value(self, location='.', skip_default=False):
+    def value(self, location=".", skip_default=False):
         """
         Return a value based on a location.
 
@@ -512,7 +534,7 @@ def value(self, location='.', skip_default=False):
 
         if skip_default:
             details = self.details(location)
-            if not details.get('history'):
+            if not details.get("history"):
                 return
 
         return value
@@ -520,7 +542,7 @@ def value(self, location='.', skip_default=False):
     # -------------------------------------------------------------------------
 
     @normalize_location
-    def details(self, location='.'):
+    def details(self, location="."):
         """
         Return details based on a location.
         """
@@ -530,15 +552,15 @@ def details(self, location='.'):
             value = container[key]
 
         if isinstance(value, PFDBObj):
-            return getattr(value, '_details_', {}).get('_value_', {})
+            return getattr(value, "_details_", {}).get("_value_", {})
 
-        key = '_value_' if key is None else key
-        return getattr(container, '_details_', {}).get(key, {})
+        key = "_value_" if key is None else key
+        return getattr(container, "_details_", {}).get(key, {})
 
     # -------------------------------------------------------------------------
 
     @normalize_location
-    def doc(self, location='.'):
+    def doc(self, location="."):
         """
         Return docs based on a location.
         """
@@ -551,11 +573,11 @@ def doc(self, location='.'):
         if not value and isinstance(container, PFDBObj):
             value = container
 
-        ret = ''
-        if isinstance(value, PFDBObj) and getattr(value, '__doc__', None):
-            ret += value.__doc__ + '\n'
-        if details.get('help'):
-            ret += details['help'] + '\n'
+        ret = ""
+        if isinstance(value, PFDBObj) and getattr(value, "__doc__", None):
+            ret += value.__doc__ + "\n"
+        if details.get("help"):
+            ret += details["help"] + "\n"
         return ret
 
     # -------------------------------------------------------------------------
@@ -566,17 +588,26 @@ def get_context_settings(self):
         This is useful when providing global information for domains or else.
         """
         return {
-            'print_line_error': settings.PRINT_LINE_ERROR,
-            'exit_on_error': settings.EXIT_ON_ERROR,
-            'working_directory': settings.WORKING_DIRECTORY,
-            'pf_version': settings.PARFLOW_VERSION
+            "print_line_error": settings.PRINT_LINE_ERROR,
+            "exit_on_error": settings.EXIT_ON_ERROR,
+            "working_directory": settings.WORKING_DIRECTORY,
+            "pf_version": settings.PARFLOW_VERSION,
         }
 
     # ---------------------------------------------------------------------------
 
-    def pfset(self, key='', value=None, yaml_file=None, yaml_content=None,
-              pfidb_file=None, hierarchical_map=None, flat_map=None,
-              exit_if_undefined=False, silence_if_undefined=False):
+    def pfset(
+        self,
+        key="",
+        value=None,
+        yaml_file=None,
+        yaml_content=None,
+        pfidb_file=None,
+        hierarchical_map=None,
+        flat_map=None,
+        exit_if_undefined=False,
+        silence_if_undefined=False,
+    ):
         """
         Allow to define any parflow key so it can be exported.
         Many formats are supported:
@@ -606,26 +637,31 @@ def pfset(self, key='', value=None, yaml_file=None, yaml_content=None,
         if flat_map:
             sorted_flat_map = sort_dict_by_priority(flat_map)
             for key, value in sorted_flat_map.items():
-                self.pfset(key=key, value=value,
-                           exit_if_undefined=exit_if_undefined,
-                           silence_if_undefined=silence_if_undefined)
+                self.pfset(
+                    key=key,
+                    value=value,
+                    exit_if_undefined=exit_if_undefined,
+                    silence_if_undefined=silence_if_undefined,
+                )
 
         if not key:
             return
 
         key_stored = False
-        tokens = key.split('.')
+        tokens = key.split(".")
         if len(tokens) > 1:
             value_key = tokens[-1]
             container = None
-            query = '/'.join(tokens[:-1])
+            query = "/".join(tokens[:-1])
             selection = self.select(query)
 
             if len(selection) > 0:
                 container = selection[0]
 
             if len(selection) > 1:
-                raise ValueError(f"Found {len(selection)} containers when selecting {query}: expected one or zero")
+                raise ValueError(
+                    f"Found {len(selection)} containers when selecting {query}: expected one or zero"
+                )
 
             if container is None:
                 # We need to maybe handle prefix
@@ -639,11 +675,10 @@ def pfset(self, key='', value=None, yaml_file=None, yaml_content=None,
                         container = container[name]
                     else:
                         # Extract available prefix
-                        known_prefixes = set('_')
+                        known_prefixes = set("_")
                         for child_name in container.keys():
                             if isinstance(container[child_name], PFDBObj):
-                                prefix = getattr(container[child_name],
-                                                 '_prefix_', '')
+                                prefix = getattr(container[child_name], "_prefix_", "")
                                 if prefix is not None:
                                     known_prefixes.add(prefix)
 
@@ -653,7 +688,7 @@ def pfset(self, key='', value=None, yaml_file=None, yaml_content=None,
                         for prefix in known_prefixes:
                             if found:
                                 break
-                            name_w_prefix = f'{prefix}{name}'
+                            name_w_prefix = f"{prefix}{name}"
                             if name_w_prefix in container.__dict__:
                                 found = True
                                 container = container[name_w_prefix]
@@ -676,17 +711,20 @@ def pfset(self, key='', value=None, yaml_file=None, yaml_content=None,
                 root = root._parent_
 
             # store key on the side
-            if '_pfstore_' not in root.__dict__:
-                root.__dict__['_pfstore_'] = {}
+            if "_pfstore_" not in root.__dict__:
+                root.__dict__["_pfstore_"] = {}
             parent_namespace = self.full_name()
-            full_key_name = f"{parent_namespace}" \
-                            f"{'.' if parent_namespace else ''}{key}"
-            root.__dict__['_pfstore_'][full_key_name] = value
+            full_key_name = (
+                f"{parent_namespace}" f"{'.' if parent_namespace else ''}{key}"
+            )
+            root.__dict__["_pfstore_"][full_key_name] = value
             root_path = self.full_name()
             if not silence_if_undefined:
-                print(f"Caution: Using internal store of "
-                      f"{root_path if root_path else 'run'} "
-                      f"to save {full_key_name} = {value}")
+                print(
+                    f"Caution: Using internal store of "
+                    f"{root_path if root_path else 'run'} "
+                    f"to save {full_key_name} = {value}"
+                )
             if exit_if_undefined:
                 sys.exit(1)
 
@@ -697,6 +735,7 @@ def _process_dynamic(self):
         Processing the dynamically defined (user-defined) key names
         """
         from . import generated
+
         for class_name, selection in self._dynamic_.items():
             klass = getattr(generated, class_name)
             names = self.select(selection)
@@ -707,7 +746,7 @@ def _process_dynamic(self):
     # -------------------------------------------------------------------------
 
     @normalize_location
-    def _value(self, location='.'):
+    def _value(self, location="."):
         """
         Internal function to get the value, container and key from location
         """
@@ -719,7 +758,7 @@ def _value(self, location='.'):
             value = getattr(container, key, None)
 
         if isinstance(value, PFDBObj):
-            value = getattr(value, '_value_', None)
+            value = getattr(value, "_value_", None)
 
         # Try to do a store lookup
         if value is None or container is None:
@@ -728,24 +767,24 @@ def _value(self, location='.'):
                 root = root._parent_
 
             if container is not None and key is not None:
-                full_key_name = '.'.join([container.full_name(), key])
-                if '_pfstore_' in root.__dict__:
-                    store = root.__dict__['_pfstore_']
+                full_key_name = ".".join([container.full_name(), key])
+                if "_pfstore_" in root.__dict__:
+                    store = root.__dict__["_pfstore_"]
                     if full_key_name in store:
                         return store[full_key_name], root, full_key_name
 
             # no container were found need to start from root
-            path_tokens = location.split('/')
+            path_tokens = location.split("/")
 
             if len(path_tokens[0]) == 0:
                 # We have abs_path
-                full_key_name = '.'.join(path_tokens[1:])
+                full_key_name = ".".join(path_tokens[1:])
             elif len(path_tokens):
                 # relative path
                 local_root = self
-                while len(path_tokens) and path_tokens[0] == '.':
+                while len(path_tokens) and path_tokens[0] == ".":
                     path_tokens.pop(0)
-                while len(path_tokens) and path_tokens[0] == '..':
+                while len(path_tokens) and path_tokens[0] == "..":
                     local_root = local_root._parent_
                     path_tokens.pop(0)
 
@@ -753,12 +792,12 @@ def _value(self, location='.'):
                 if prefix_name:
                     path_tokens.insert(0, prefix_name)
 
-                full_key_name = '.'.join(path_tokens)
+                full_key_name = ".".join(path_tokens)
 
             # Resolve full_key_name from root
             current_key = full_key_name
             current_node = root
-            current_store = getattr(current_node, '_pfstore_', None)
+            current_store = getattr(current_node, "_pfstore_", None)
 
             # Find key in store
             while len(current_key) and current_store is not None:
@@ -767,12 +806,15 @@ def _value(self, location='.'):
 
                 # Find child store
                 current_store = None
-                while (len(current_key) > 0 and current_node is not None and
-                        current_store is None):
-                    tokens = current_key.split('.')
+                while (
+                    len(current_key) > 0
+                    and current_node is not None
+                    and current_store is None
+                ):
+                    tokens = current_key.split(".")
                     current_node = current_node[tokens[0]]
-                    current_store = getattr(current_node, '_pfstore_', None)
-                    current_key = '.'.join(tokens[1:])
+                    current_store = getattr(current_node, "_pfstore_", None)
+                    current_key = ".".join(tokens[1:])
 
         return value, container, key
 
@@ -783,17 +825,17 @@ def _get_container_and_key(self, location):
         """
         Internal function to get the container and key from location
         """
-        split = location.split('/')
-        path, key = '/'.join(split[:-1]), split[-1]
+        split = location.split("/")
+        path, key = "/".join(split[:-1]), split[-1]
 
-        if key == '.':
+        if key == ".":
             return self, None
 
         parent = self
-        while key == '..' and getattr(parent, '_parent_', None):
+        while key == ".." and getattr(parent, "_parent_", None):
             parent = parent._parent_
-            split = path.split('/')
-            path, key = '/'.join(split[:-1]), split[-1]
+            split = path.split("/")
+            path, key = "/".join(split[:-1]), split[-1]
 
         if parent is not self:
             # We went through the loop above. Return...
@@ -810,12 +852,12 @@ def _get_container_and_key(self, location):
 # Main DB Object
 # -----------------------------------------------------------------------------
 
+
 class PFDBObjListNumber(PFDBObj):
     """Class for leaf list values"""
 
     def __setattr__(self, name, value):
-        """Helper method that aims to streamline dot notation assignment
-        """
+        """Helper method that aims to streamline dot notation assignment"""
         key_str = str(name)
         if is_private_key(key_str):
             self.__dict__[key_str] = value
@@ -825,7 +867,7 @@ def __setattr__(self, name, value):
             if key_str.startswith(self._prefix_):
                 self.__dict__[key_str] = value
             else:
-                self.__dict__[f'{self._prefix_}{key_str}'] = value
+                self.__dict__[f"{self._prefix_}{key_str}"] = value
             return
 
         self.__dict__[key_str] = value
@@ -835,5 +877,5 @@ def to_pf_name(self, parent_namespace, key):
         a given field key. This allows handling of differences
         between what can be defined in Python vs Parflow key.
         """
-        start = f'{parent_namespace}.' if parent_namespace else ''
+        start = f"{parent_namespace}." if parent_namespace else ""
         return start + remove_prefix(key, self._prefix_)
diff --git a/pftools/python/parflow/tools/database/domains.py b/pftools/python/parflow/tools/database/domains.py
index 007a3c3a0..4e7942039 100644
--- a/pftools/python/parflow/tools/database/domains.py
+++ b/pftools/python/parflow/tools/database/domains.py
@@ -16,33 +16,31 @@
 # Validation helper functions
 # -----------------------------------------------------------------------------
 
+
 def filter_errors_by_type(msg_type, errors):
     """Extract decorated message for a given type"""
-    return list(filter(lambda e: e['type'] == msg_type, errors))
+    return list(filter(lambda e: e["type"] == msg_type, errors))
 
 
 # -----------------------------------------------------------------------------
 
+
 def error(message):
     """Message decorator that add ERROR as type"""
-    return {
-        'type': 'ERROR',
-        'message': message
-    }
+    return {"type": "ERROR", "message": message}
 
 
 # -----------------------------------------------------------------------------
 
+
 def warning(message):
     """Message decorator that add WARNING as type"""
-    return {
-        'type': 'WARNING',
-        'message': message
-    }
+    return {"type": "WARNING", "message": message}
 
 
 # -----------------------------------------------------------------------------
 
+
 def get_comparable_version(version):
     """Return an integer that can be used for comparison
 
@@ -54,8 +52,8 @@ def get_comparable_version(version):
 
     """
     c_version = 0
-    valid_version_number = version[1:] if version[0] == 'v' else version
-    version_tokens = valid_version_number.split('.')
+    valid_version_number = version[1:] if version[0] == "v" else version
+    version_tokens = valid_version_number.split(".")
     for version_token in version_tokens:
         c_version *= 1000
         c_version += int(version_token)
@@ -64,6 +62,7 @@ def get_comparable_version(version):
 
 # -----------------------------------------------------------------------------
 
+
 def get_installed_parflow_module(module):
     """Helper function to test if a given module is available inside ParFlow
     This method rely on PARFLOW_DIR environment variable.
@@ -80,10 +79,10 @@ def get_installed_parflow_module(module):
     if module_file.resolve().exists():
         with open(module_file, "r") as f:
             for line in f:
-                if f'PARFLOW_HAVE_{module}' in line and 'yes' in line:
+                if f"PARFLOW_HAVE_{module}" in line and "yes" in line:
                     has_module_installed = True
     else:
-        print(f'Cannot find Makefile.config in {str(module_file.resolve())}.')
+        print(f"Cannot find Makefile.config in {str(module_file.resolve())}.")
     return has_module_installed
 
 
@@ -91,15 +90,18 @@ def get_installed_parflow_module(module):
 # Validation classes
 # -----------------------------------------------------------------------------
 
+
 class ValidationException(Exception):
     """
     Basic parflow exception used for domains to report error
     """
+
     pass
 
 
 # -----------------------------------------------------------------------------
 
+
 class MandatoryValue:
     """
     MandatoryValue makes sure that the key is set
@@ -108,10 +110,11 @@ class MandatoryValue:
     the default value will be written out in the validation message
     and database file.
     """
+
     def validate(self, value, **kwargs):
         errors = []
         if value is None:
-            errors.append(error('Needs to be set'))
+            errors.append(error("Needs to be set"))
 
         return errors
 
@@ -126,6 +129,7 @@ class IntValue:
       - min_value: If available the value must be strictly above it
       - max_value: If available the value must be strictly below it
     """
+
     def validate(self, value, min_value=None, max_value=None, **kwargs):
         errors = []
 
@@ -133,12 +137,12 @@ def validate(self, value, min_value=None, max_value=None, **kwargs):
             return errors
 
         if not isinstance(value, int):
-            errors.append(error('Needs to be an integer'))
+            errors.append(error("Needs to be an integer"))
 
         if min_value is not None and value < min_value:
-            errors.append(error(f'Is smaller than min: {min_value}'))
+            errors.append(error(f"Is smaller than min: {min_value}"))
         if max_value is not None and value > max_value:
-            errors.append(error(f'Is greater than max: {max_value}'))
+            errors.append(error(f"Is greater than max: {max_value}"))
 
         return errors
 
@@ -153,6 +157,7 @@ class DoubleValue:
       - min_value: If available the value must be strictly above it
       - max_value: If available the value must be strictly below it
     """
+
     def validate(self, value, min_value=None, max_value=None, **kwargs):
         errors = []
 
@@ -160,15 +165,16 @@ def validate(self, value, min_value=None, max_value=None, **kwargs):
             return errors
 
         if not isinstance(value, (float, int)):
-            errors.append(error('Needs to be a double'))
+            errors.append(error("Needs to be a double"))
 
         if min_value is not None and value < min_value:
-            errors.append(error(f'Is smaller than min: {min_value}'))
+            errors.append(error(f"Is smaller than min: {min_value}"))
         if max_value is not None and value > max_value:
-            errors.append(error(f'Is greater than max: {max_value}'))
+            errors.append(error(f"Is greater than max: {max_value}"))
 
         return errors
 
+
 # -----------------------------------------------------------------------------
 
 
@@ -179,7 +185,17 @@ class EnumDomain:
 
     The expected keyword argument is a list of the accepted values.
     """
-    def validate(self, value, enum_list=[], location='', locations=[], container=None,  pf_version=None, **kwargs):
+
+    def validate(
+        self,
+        value,
+        enum_list=[],
+        location="",
+        locations=[],
+        container=None,
+        pf_version=None,
+        **kwargs,
+    ):
         errors = []
 
         if value is None:
@@ -194,8 +210,7 @@ def validate(self, value, enum_list=[], location='', locations=[], container=Non
 
         if isinstance(enum_list, dict):
             # We need to find the matching version
-            sorted_versions = [
-                (get_comparable_version(v), v) for v in enum_list.keys()]
+            sorted_versions = [(get_comparable_version(v), v) for v in enum_list.keys()]
             sorted_versions.sort(key=lambda t: t[0])
             version_to_use = sorted_versions[0]
             current_version = get_comparable_version(pf_version)
@@ -215,15 +230,16 @@ def validate(self, value, enum_list=[], location='', locations=[], container=Non
         if isinstance(value, list):
             for v in value:
                 if v not in lookup_list:
-                    str_list = ', '.join(lookup_list)
-                    errors.append(error(f'{v} must be one of [{str_list}]'))
-        else: 
+                    str_list = ", ".join(lookup_list)
+                    errors.append(error(f"{v} must be one of [{str_list}]"))
+        else:
             if value not in lookup_list:
-                str_list = ', '.join(lookup_list)
-                errors.append(error(f'{value} must be one of [{str_list}]'))
+                str_list = ", ".join(lookup_list)
+                errors.append(error(f"{value} must be one of [{str_list}]"))
 
         return errors
 
+
 # -----------------------------------------------------------------------------
 
 
@@ -231,6 +247,7 @@ class AnyString:
     """
     AnyString domain constrains the value to be a string or list of strings.
     """
+
     def validate(self, value, **kwargs):
         errors = []
 
@@ -240,9 +257,10 @@ def validate(self, value, **kwargs):
         if isinstance(value, (list, str)):
             return errors
 
-        errors.append(error(f'{value} ({type(value)} must be a string'))
+        errors.append(error(f"{value} ({type(value)} must be a string"))
         return errors
 
+
 # -----------------------------------------------------------------------------
 
 
@@ -250,6 +268,7 @@ class BoolDomain:
     """
     BoolDomain domain constrains the value to be a boolean.
     """
+
     def validate(self, value, **kwargs):
         errors = []
 
@@ -259,9 +278,10 @@ def validate(self, value, **kwargs):
         if isinstance(value, bool):
             return errors
 
-        errors.append(error(f'{value} ({type(value)} must be True/False)'))
+        errors.append(error(f"{value} ({type(value)} must be True/False)"))
         return errors
 
+
 # -----------------------------------------------------------------------------
 
 
@@ -269,31 +289,39 @@ class ValidFile:
     """
     ValidFile domain checks the working directory to find the specified file.
     """
-    def validate(self, value, working_directory=None, path_prefix_source=None,
-                 container=None, **kwargs):
+
+    def validate(
+        self,
+        value,
+        working_directory=None,
+        path_prefix_source=None,
+        container=None,
+        **kwargs,
+    ):
         errors = []
-        path_prefix = ''
+        path_prefix = ""
 
         if value is None:
             return errors
 
         if working_directory is None:
-            errors.append(error('Working directory is not defined'))
+            errors.append(error("Working directory is not defined"))
             return errors
 
         if path_prefix_source:
-            path_prefix, = container.select(path_prefix_source)
+            (path_prefix,) = container.select(path_prefix_source)
 
         path = Path(working_directory) / path_prefix / value
         if path.exists():
             return errors
 
-        errors.append(error(f'Could not locate file {str(path.resolve())}'))
+        errors.append(error(f"Could not locate file {str(path.resolve())}"))
         return errors
 
 
 # -----------------------------------------------------------------------------
 
+
 class AddedInVersion:
     """
     AddedInVersion domain deals with keys that were added to the ParFlow code
@@ -301,6 +329,7 @@ class AddedInVersion:
     version and print an error if your ParFlow version does not have the given
     key.
     """
+
     def validate(self, value, arg, pf_version=None, **kwargs):
         errors = []
 
@@ -311,14 +340,14 @@ def validate(self, value, arg, pf_version=None, **kwargs):
         current_version = get_comparable_version(pf_version)
 
         if version > current_version:
-            errors.append(
-                error(f'Not valid in ParFlow versions before v{arg}'))
+            errors.append(error(f"Not valid in ParFlow versions before v{arg}"))
 
         return errors
 
 
 # -----------------------------------------------------------------------------
 
+
 class DeprecatedInVersion:
     """
     DeprecatedInVersion domain deals with keys that have been or will be
@@ -326,6 +355,7 @@ class DeprecatedInVersion:
     version and print an error or warning depending on whether the key has
     been deprecated.
     """
+
     def validate(self, value, arg, pf_version=None, **kwargs):
         errors = []
 
@@ -336,13 +366,14 @@ def validate(self, value, arg, pf_version=None, **kwargs):
         current_version = get_comparable_version(pf_version)
 
         if version <= current_version:
-            errors.append(error(f'Deprecated in v{arg}'))
+            errors.append(error(f"Deprecated in v{arg}"))
 
         if version > current_version:
-            errors.append(warning(f'Will be deprecated in v{arg}'))
+            errors.append(warning(f"Will be deprecated in v{arg}"))
 
         return errors
 
+
 # -----------------------------------------------------------------------------
 
 
@@ -353,6 +384,7 @@ class RemovedInVersion:
     removed version and print an error or warning depending on whether the key
     has been or will be removed.
     """
+
     def validate(self, value, arg, pf_version=None, **kwargs):
         errors = []
 
@@ -363,13 +395,14 @@ def validate(self, value, arg, pf_version=None, **kwargs):
         current_version = get_comparable_version(pf_version)
 
         if version <= current_version:
-            errors.append(error(f'Removed in v{arg}'))
+            errors.append(error(f"Removed in v{arg}"))
 
         if version > current_version:
-            errors.append(warning(f'Will be removed in v{arg}'))
+            errors.append(warning(f"Will be removed in v{arg}"))
 
         return errors
 
+
 # -----------------------------------------------------------------------------
 
 
@@ -380,6 +413,7 @@ class RequiresModule:
     see whether the required modules are installed with ParFlow and will print
     an error message if the required module is missing.
     """
+
     def validate(self, value, arg, **kwargs):
         errors = []
 
@@ -390,7 +424,7 @@ def validate(self, value, arg, **kwargs):
 
         for module in arg_list:
             if not get_installed_parflow_module(module):
-                errors.append(error(f'Need to install {module} module'))
+                errors.append(error(f"Need to install {module} module"))
 
         return errors
 
@@ -418,8 +452,10 @@ def get_domain(class_name):
         AVAILABLE_DOMAINS[class_name] = instance
         return instance
 
-    print(f'{term.FAIL}{term_symbol.ko}{term.ENDC} Could not find domain: '
-          f'"{class_name}"')
+    print(
+        f"{term.FAIL}{term_symbol.ko}{term.ENDC} Could not find domain: "
+        f'"{class_name}"'
+    )
 
     return None
 
@@ -428,8 +464,10 @@ def get_domain(class_name):
 # API meant to be used outside of this module
 # -----------------------------------------------------------------------------
 
-def validate_value_with_errors(value, domain_definitions=None,
-                               domain_add_on_kwargs=None):
+
+def validate_value_with_errors(
+    value, domain_definitions=None, domain_add_on_kwargs=None
+):
     """This method validates the value set to a key using the domains
     provided in the key definition files.
 
@@ -464,9 +502,9 @@ def validate_value_with_errors(value, domain_definitions=None,
 
             if domain_definitions[domain_classname]:
                 if isinstance(domain_definitions[domain_classname], str):
-                    domain_kwargs['arg'] = domain_definitions[domain_classname]
+                    domain_kwargs["arg"] = domain_definitions[domain_classname]
                 elif isinstance(domain_definitions[domain_classname], list):
-                    domain_kwargs['arg'] = domain_definitions[domain_classname]
+                    domain_kwargs["arg"] = domain_definitions[domain_classname]
                 else:
                     domain_kwargs.update(domain_definitions[domain_classname])
 
@@ -474,12 +512,13 @@ def validate_value_with_errors(value, domain_definitions=None,
 
     return errors
 
+
 # -----------------------------------------------------------------------------
 
 
-def validate_value_with_exception(value, domain_definition=None,
-                                  domain_add_on_kwargs=None,
-                                  exit_on_error=False):
+def validate_value_with_exception(
+    value, domain_definition=None, domain_add_on_kwargs=None, exit_on_error=False
+):
     """This method validates the value set to a key using the domains
     provided in the key definition files. But it will print information
     on where the error was detected (line number).
@@ -502,8 +541,9 @@ def validate_value_with_exception(value, domain_definition=None,
 
     """
     all_messages = validate_value_with_errors(
-        value, domain_definition, domain_add_on_kwargs)
-    errors = filter_errors_by_type('ERROR', all_messages)
+        value, domain_definition, domain_add_on_kwargs
+    )
+    errors = filter_errors_by_type("ERROR", all_messages)
 
     if errors:
         print()
@@ -513,13 +553,13 @@ def validate_value_with_exception(value, domain_definition=None,
             exp, val, tb = sys.exc_info()
             listing = traceback.format_stack(tb.tb_frame)
             for item in listing:
-                if 'parflow/database' in item:
+                if "parflow/database" in item:
                     continue
                 print(item)
 
-        print(f'    The value {value} is invalid')
+        print(f"    The value {value} is invalid")
         for error in errors:
-            print(f'    - {error}')
+            print(f"    - {error}")
         print()
 
         if exit_on_error:
@@ -528,9 +568,16 @@ def validate_value_with_exception(value, domain_definition=None,
 
 # -----------------------------------------------------------------------------
 
-def validate_value_to_string(container, value, has_default=False,
-                             domain_definition=None, domain_add_on_kwargs=None,
-                             history=None, indent=1):
+
+def validate_value_to_string(
+    container,
+    value,
+    has_default=False,
+    domain_definition=None,
+    domain_add_on_kwargs=None,
+    history=None,
+    indent=1,
+):
     """This method validates the value set to a key using the domains
     provided in the key definition files. But it will return a string
     that could be used for printing information.
@@ -557,44 +604,52 @@ def validate_value_to_string(container, value, has_default=False,
         number_of_errors (int): Number of detected issue
         message (str): String to print
     """
-    indent_str = '  ' * (indent - 1)
-    domain_add_on_kwargs['container'] = container
+    indent_str = "  " * (indent - 1)
+    domain_add_on_kwargs["container"] = container
     all_messages = validate_value_with_errors(
-        value, domain_definition, domain_add_on_kwargs)
-    errors = filter_errors_by_type('ERROR', all_messages)
-    warnings = filter_errors_by_type('WARNING', all_messages)
+        value, domain_definition, domain_add_on_kwargs
+    )
+    errors = filter_errors_by_type("ERROR", all_messages)
+    warnings = filter_errors_by_type("WARNING", all_messages)
     validation_string = []
 
     if errors:
-        validation_string.append(
-            f'{value} {term.FAIL}{term_symbol.ko}{term.ENDC}')
+        validation_string.append(f"{value} {term.FAIL}{term_symbol.ko}{term.ENDC}")
         for error in errors:
-            validation_string.append(f'{indent_str}    {term.WARNING}'
-                                     f'{term_symbol.errorItem}{term.ENDC} '
-                                     f'{error}')
+            validation_string.append(
+                f"{indent_str}    {term.WARNING}"
+                f"{term_symbol.errorItem}{term.ENDC} "
+                f"{error}"
+            )
     elif value is not None:
         # checking for duplicates and changing print statement
         if history is not None:
             dup_count = len(history) - 1 if has_default else len(history)
             if dup_count > 1:
-                dup_str = '('
+                dup_str = "("
                 for val in range(dup_count - 1):
-                    dup_str += str(history[val]) + ' => '
-                dup_str += str(history[dup_count - 1]) + ')'
-                validation_string.append(f'{term.MAGENTA}{term_symbol.warning}'
-                                         f'{term.ENDC} {value}  '
-                                         f'{term.MAGENTA}{dup_str}{term.ENDC}')
+                    dup_str += str(history[val]) + " => "
+                dup_str += str(history[dup_count - 1]) + ")"
+                validation_string.append(
+                    f"{term.MAGENTA}{term_symbol.warning}"
+                    f"{term.ENDC} {value}  "
+                    f"{term.MAGENTA}{dup_str}{term.ENDC}"
+                )
             else:
-                validation_string.append(f'{value} {term.OKGREEN}'
-                                         f'{term_symbol.ok}{term.ENDC}')
+                validation_string.append(
+                    f"{value} {term.OKGREEN}" f"{term_symbol.ok}{term.ENDC}"
+                )
         else:
-            validation_string.append(f'{value} {term.OKGREEN}{term_symbol.ok}'
-                                     f'{term.ENDC}')
+            validation_string.append(
+                f"{value} {term.OKGREEN}{term_symbol.ok}" f"{term.ENDC}"
+            )
 
     if warnings:
         for warning in warnings:
-            validation_string.append(f'{indent_str}    {term.CYAN}'
-                                     f'{term_symbol.warning}{term.ENDC} '
-                                     f'{warning}')
+            validation_string.append(
+                f"{indent_str}    {term.CYAN}"
+                f"{term_symbol.warning}{term.ENDC} "
+                f"{warning}"
+            )
 
-    return len(all_messages), '\n'.join(validation_string)
+    return len(all_messages), "\n".join(validation_string)
diff --git a/pftools/python/parflow/tools/database/handlers.py b/pftools/python/parflow/tools/database/handlers.py
index 3be410c82..66c250042 100644
--- a/pftools/python/parflow/tools/database/handlers.py
+++ b/pftools/python/parflow/tools/database/handlers.py
@@ -19,18 +19,22 @@ class ValueHandlerException(Exception):
     """
     Basic parflow exception used for ValueHandlers to report error
     """
+
     pass
 
 
 # -----------------------------------------------------------------------------
 
+
 class ChildHandler:
     """
     This class creates new keys from user-defined name input
     (e.g. GeomName)
     """
-    def decorate(self, value, container, class_name=None, location='.',
-                 eager=None, **kwargs):
+
+    def decorate(
+        self, value, container, class_name=None, location=".", eager=None, **kwargs
+    ):
 
         klass = None
         destination_containers = []
@@ -45,32 +49,37 @@ def decorate(self, value, container, class_name=None, location='.',
         for destination_container in destination_containers:
             if destination_container is not None:
                 if valid_name not in destination_container.__dict__:
-                    destination_container.__dict__[valid_name] = (
-                        klass(destination_container))
+                    destination_container.__dict__[valid_name] = klass(
+                        destination_container
+                    )
                 elif eager:
-                    print(f'Error no selection for {location}')
+                    print(f"Error no selection for {location}")
 
         return valid_name
 
 
 # -----------------------------------------------------------------------------
 
+
 class ChildrenHandler:
     """
     This class creates new keys from user-defined name inputs
     (e.g. GeomNames)
     """
+
     def __init__(self):
         self.child_handler = ChildHandler()
 
-    def decorate(self, value, container, class_name=None, location='.',
-                 eager=None, **kwargs):
+    def decorate(
+        self, value, container, class_name=None, location=".", eager=None, **kwargs
+    ):
         if isinstance(value, str):
             names = value.split()
             valid_names = []
             for name in names:
                 valid_name = self.child_handler.decorate(
-                    name, container, class_name, location, eager)
+                    name, container, class_name, location, eager
+                )
                 if valid_name is not None:
                     valid_names.append(valid_name)
 
@@ -81,45 +90,56 @@ def decorate(self, value, container, class_name=None, location='.',
         elif isinstance(value, int):
             valid_names = []
             for i in range(value):
-                name = f'_{i}'  # FIXME should use prefix instead
+                name = f"_{i}"  # FIXME should use prefix instead
                 valid_names.append(
-                    self.child_handler.decorate(name, container, class_name,
-                                                location, eager))
+                    self.child_handler.decorate(
+                        name, container, class_name, location, eager
+                    )
+                )
 
             return valid_names
 
-        if hasattr(value, '__iter__'):
+        if hasattr(value, "__iter__"):
             valid_names = []
             for name in value:
                 valid_name = self.child_handler.decorate(
-                    name, container, class_name, location, eager)
+                    name, container, class_name, location, eager
+                )
                 if valid_name is not None:
                     valid_names.append(valid_name)
 
             return valid_names
 
         raise ValueHandlerException(
-            f'{value} is not of the expected type for ChildrenHandler')
+            f"{value} is not of the expected type for ChildrenHandler"
+        )
+
 
 # -----------------------------------------------------------------------------
 class ListHandler:
     """
     This class ensures the output is not a single string but a list of trimmed string.
     """
+
     def __init__(self):
         self.children_handler = ChildrenHandler()
 
     def decorate(self, value, container, **kwargs):
         return self.children_handler.decorate(value, container)
 
+
 # -----------------------------------------------------------------------------
 
+
 class SplitHandler:
     """
     This class will split the provided key using the separator convert
     each tocken using a type converter and set the field list.
     """
-    def decorate(self, value, container, separator='/', convert='int', fields=None, **kwargs):
+
+    def decorate(
+        self, value, container, separator="/", convert="int", fields=None, **kwargs
+    ):
         if isinstance(value, str):
             tokens = value.split(separator)
             index = 0
@@ -132,7 +152,9 @@ def decorate(self, value, container, separator='/', convert='int', fields=None,
             return value
 
         raise ValueHandlerException(
-            f'{value} is not of the expected type for SplitHandler')
+            f"{value} is not of the expected type for SplitHandler"
+        )
+
 
 # -----------------------------------------------------------------------------
 # Helper map with an instance of each Value handler
@@ -160,8 +182,10 @@ def get_handler(class_name, print_error=True):
         return instance
 
     if print_error:
-        print(f'{term.FAIL}{term_symbol.ko}{term.ENDC} Could not find '
-              f'handler: "{class_name}"')
+        print(
+            f"{term.FAIL}{term_symbol.ko}{term.ENDC} Could not find "
+            f'handler: "{class_name}"'
+        )
 
     return None
 
@@ -170,6 +194,7 @@ def get_handler(class_name, print_error=True):
 # API meant to be used outside of this module
 # -----------------------------------------------------------------------------
 
+
 def decorate_value(value, container=None, handlers=None):
     """Return a decorated version of the value while
     possibly affecting other keys by creating children.
@@ -207,8 +232,8 @@ def decorate_value(value, container=None, handlers=None):
     for handler_classname in handlers:
         handler = get_handler(handler_classname, False)
 
-        if handler is None and 'type' in handlers[handler_classname]:
-            handler = get_handler(handlers[handler_classname]['type'])
+        if handler is None and "type" in handlers[handler_classname]:
+            handler = get_handler(handlers[handler_classname]["type"])
         else:
             get_handler(handler_classname)
 
@@ -217,8 +242,7 @@ def decorate_value(value, container=None, handlers=None):
             if isinstance(handler_kwargs, str):
                 return_value = handler.decorate(value, container)
             else:
-                return_value = handler.decorate(
-                    value, container, **handler_kwargs)
+                return_value = handler.decorate(value, container, **handler_kwargs)
 
     # added to handle variable DZ
     if isinstance(value, int):
diff --git a/pftools/python/parflow/tools/export.py b/pftools/python/parflow/tools/export.py
index 09622736e..aaaf0e0aa 100644
--- a/pftools/python/parflow/tools/export.py
+++ b/pftools/python/parflow/tools/export.py
@@ -19,7 +19,7 @@
 
 
 # For instance: [Type: double]
-TYPE_INFO_RE = re.compile(r'(^\[\w+: \w+\] )')
+TYPE_INFO_RE = re.compile(r"(^\[\w+: \w+\] )")
 
 
 class SubsurfacePropertiesExporter:
@@ -28,21 +28,21 @@ def __init__(self, run):
         self.run = run
         self.props_found = set()
         self.entries = []
-        yaml_key_def = Path(__file__).parent / 'ref/table_keys.yaml'
-        with open(yaml_key_def, 'r') as file:
+        yaml_key_def = Path(__file__).parent / "ref/table_keys.yaml"
+        with open(yaml_key_def, "r") as file:
             self.definition = yaml.safe_load(file)
 
         self.pfkey_to_alias = {}
         self.alias_to_priority = {}
         for i, (key, value) in enumerate(self.definition.items()):
-            self.pfkey_to_alias[key] = value['alias'][0]
-            self.alias_to_priority[value['alias'][0]] = i
+            self.pfkey_to_alias[key] = value["alias"][0]
+            self.alias_to_priority[value["alias"][0]] = i
 
         self._process()
 
     def _extract_sub_surface_props(self, geom_item):
         name = Path(geom_item.full_name()).suffix[1:]
-        entry = {'key': name}
+        entry = {"key": name}
         has_data = False
         for key in self.pfkey_to_alias:
             value = geom_item.value(key, skip_default=True)
@@ -57,14 +57,14 @@ def _extract_sub_surface_props(self, geom_item):
     def _process(self):
         self.entries = []
         self.props_found.clear()
-        geom_items = self.run.Geom.select('{GeomItem}')
+        geom_items = self.run.Geom.select("{GeomItem}")
         for item in geom_items:
             entry = self._extract_sub_surface_props(item)
             if entry is not None:
                 self.entries.append(entry)
 
-    def get_table_as_txt(self, column_separator='  ', columns_justify=True):
-        header = ['key'] + list(self.props_found)
+    def get_table_as_txt(self, column_separator="  ", columns_justify=True):
+        header = ["key"] + list(self.props_found)
         header.sort(key=lambda alias: self.alias_to_priority[alias])
         lines = []
 
@@ -74,7 +74,7 @@ def get_table_as_txt(self, column_separator='  ', columns_justify=True):
             if columns_justify:
                 sizes[key] = len(key)
                 for entry in self.entries:
-                    value = entry[key] if key in entry else '-'
+                    value = entry[key] if key in entry else "-"
                     sizes[key] = max(sizes[key], len(value))
             else:
                 sizes[key] = 0
@@ -89,20 +89,19 @@ def get_table_as_txt(self, column_separator='  ', columns_justify=True):
         for entry in self.entries:
             line = []
             for key in header:
-                value = entry[key] if key in entry else '-'
+                value = entry[key] if key in entry else "-"
                 line.append(value.ljust(sizes[key]))
             lines.append(column_separator.join(line))
 
-        return '\n'.join(lines)
+        return "\n".join(lines)
 
     def write_csv(self, file_path):
-        data = self.get_table_as_txt(column_separator=',',
-                                     columns_justify=False)
-        Path(file_path).write_text(data, encoding='utf-8')
+        data = self.get_table_as_txt(column_separator=",", columns_justify=False)
+        Path(file_path).write_text(data, encoding="utf-8")
 
     def write_txt(self, file_path):
         data = self.get_table_as_txt()
-        Path(file_path).write_text(data, encoding='utf-8')
+        Path(file_path).write_text(data, encoding="utf-8")
 
 
 class CLMExporter:
@@ -110,34 +109,34 @@ class CLMExporter:
     def __init__(self, run):
         self.run = run
 
-    def write(self, working_directory='.', **kwargs):
+    def write(self, working_directory=".", **kwargs):
         """Method to export all clm files based on metadata
 
         Args:
             - working_directory='.': specifies where the files
               will be written
         """
-        kwargs['working_directory'] = working_directory
+        kwargs["working_directory"] = working_directory
         self.write_input(**kwargs)
         self.write_map(**kwargs)
         self.write_parameters(**kwargs)
 
-    def write_allowed(self, working_directory='.', **kwargs):
+    def write_allowed(self, working_directory=".", **kwargs):
         """Export files we are allowed to overwrite
 
         Args:
             - working_directory='.': specifies where the files
               will be written
         """
-        kwargs['working_directory'] = working_directory
+        kwargs["working_directory"] = working_directory
         to_write = [
-            'input',
-            'map',
-            'parameters',
+            "input",
+            "map",
+            "parameters",
         ]
 
         for name in to_write:
-            func = getattr(self, f'write_{name}')
+            func = getattr(self, f"write_{name}")
             try:
                 func(**kwargs)
             except NotOverwritableException:
@@ -146,25 +145,25 @@ def write_allowed(self, working_directory='.', **kwargs):
                     # in the calculation
                     self._print_not_written_warning(working_directory, name)
 
-    def write_input(self, working_directory='.'):
+    def write_input(self, working_directory="."):
         """Method to export drv_clmin.dat file based on metadata
 
         Args:
             - working_directory='.': specifies where drv_climin.dat
               file will be written
         """
-        output_file = self._file(working_directory, 'input')
+        output_file = self._file(working_directory, "input")
 
         # Make sure we don't overwrite pre-existing data
-        self._ensure_writable(output_file, 'input')
+        self._ensure_writable(output_file, "input")
 
         clm_drv_keys = {}
-        header_doc = ''
+        header_doc = ""
         clm_dict = self._clm_solver.Input.to_dict()
 
         required_input_keys = [
-            'File.VegTileSpecification',
-            'File.VegTypeParameter',
+            "File.VegTileSpecification",
+            "File.VegTypeParameter",
         ]
 
         # Ensure required input keys are set
@@ -173,41 +172,42 @@ def write_input(self, working_directory='.'):
                 continue
 
             # Set the default for this key
-            default = self._clm_solver.Input.details(key).get('default')
+            default = self._clm_solver.Input.details(key).get("default")
             clm_dict[key] = default
 
         # Gather doc information to print out
         for key in clm_dict:
             old_header_doc = header_doc
-            container_key = '.'.join(map(str, key.split('.')[:-1]))
+            container_key = ".".join(map(str, key.split(".")[:-1]))
             header_doc = self._clm_solver.Input.doc(container_key)
-            clm_key = self._clm_solver.Input.details(key).get('clm_key')
+            clm_key = self._clm_solver.Input.details(key).get("clm_key")
             clm_key_value = self._clm_solver.Input.value(key)
             clm_key_help = self._clm_solver.Input.doc(key)
-            item = {'value': clm_key_value, 'help': clm_key_help}
+            item = {"value": clm_key_value, "help": clm_key_help}
 
             clm_drv_keys.setdefault(container_key, {})[clm_key] = item
             if header_doc != old_header_doc:
-                clm_drv_keys[container_key]['doc'] = header_doc
+                clm_drv_keys[container_key]["doc"] = header_doc
 
-        with open(output_file, 'w') as wf:
-            wf.write(f'! {self._auto_generated_file_string}\n')
-            wf.write(f'! CLM input file for {self.run.get_name()} '
-                     f'ParFlow run' + '\n')
+        with open(output_file, "w") as wf:
+            wf.write(f"! {self._auto_generated_file_string}\n")
+            wf.write(
+                f"! CLM input file for {self.run.get_name()} " f"ParFlow run" + "\n"
+            )
             for key, value in clm_drv_keys.items():
-                doc = str(clm_drv_keys[key]['doc']).strip().replace('\n', ' ')
-                wf.write('!\n')
-                wf.write(f'! {doc}\n')
-                wf.write('!\n')
+                doc = str(clm_drv_keys[key]["doc"]).strip().replace("\n", " ")
+                wf.write("!\n")
+                wf.write(f"! {doc}\n")
+                wf.write("!\n")
                 for sub_key, sub_value in value.items():
-                    if sub_key == 'doc':
+                    if sub_key == "doc":
                         continue
 
-                    value = sub_value['value']
-                    help = sub_value['help'].replace('\n', ' ')
-                    line = f'{sub_key:<15}'
-                    line += f'{value:<40}'
-                    line += f'{help}\n\n'
+                    value = sub_value["value"]
+                    help = sub_value["help"].replace("\n", " ")
+                    line = f"{sub_key:<15}"
+                    line += f"{value:<40}"
+                    line += f"{help}\n\n"
                     wf.write(line)
 
         return self
@@ -229,63 +229,70 @@ def _process_vegm(self, token, x, y, axis=None):
         else:
             vegm_root_key = self._clm_solver.Vegetation.Map[token]
         array = np.zeros((x, y))
-        if vegm_root_key.Type == 'Constant':
+        if vegm_root_key.Type == "Constant":
             array = np.full((x, y), vegm_root_key.Value)
-        elif vegm_root_key.Type == 'Linear':
+        elif vegm_root_key.Type == "Linear":
             min_par = vegm_root_key.Min
             max_par = vegm_root_key.Max
-            length = y if axis == 'y' else x
+            length = y if axis == "y" else x
             inc = (max_par - min_par) / (length - 1)
             list_par = list(np.arange(min_par, max_par + inc, inc))
             for i, v in enumerate(list_par):
-                if axis == 'y':
+                if axis == "y":
                     array[:, i] = v
-                elif axis == 'x':
+                elif axis == "x":
                     array[i, :] = v
                 else:
-                    raise Exception(f'Axis specification error: {axis}')
-        elif vegm_root_key.Type == 'PFBFile':
+                    raise Exception(f"Axis specification error: {axis}")
+        elif vegm_root_key.Type == "PFBFile":
             array = read_pfb(get_absolute_path(vegm_root_key.FileName))
             while array.ndim > 2:
                 # PFB files return 3D arrays, but the data is actually 2D
                 array = array[0]
         else:
-            raise Exception(f'Unknown vegm type: {vegm_root_key.Type}')
+            raise Exception(f"Unknown vegm type: {vegm_root_key.Type}")
 
         return array
 
-    def _process_vegm_loc(self, vegm_array, latitude=True, lat_axis='y',
-                          longitude=True, long_axis='x'):
+    def _process_vegm_loc(
+        self, vegm_array, latitude=True, lat_axis="y", longitude=True, long_axis="x"
+    ):
 
         # Need to better expose the options of which axis to use
         # Maybe have it as an extra key?
         x, y, z = vegm_array.shape
         if latitude:
-            vegm_array[:, :, 0] = self._process_vegm('Latitude', x, y,
-                                                     lat_axis)
+            vegm_array[:, :, 0] = self._process_vegm("Latitude", x, y, lat_axis)
 
         if longitude:
-            vegm_array[:, :, 1] = self._process_vegm('Longitude', x, y,
-                                                     long_axis)
-
-    def _process_vegm_soil(self, vegm_array, sand=True, sand_axis='y',
-                           clay=True, clay_axis='y', color=True,
-                           color_axis='y'):
+            vegm_array[:, :, 1] = self._process_vegm("Longitude", x, y, long_axis)
+
+    def _process_vegm_soil(
+        self,
+        vegm_array,
+        sand=True,
+        sand_axis="y",
+        clay=True,
+        clay_axis="y",
+        color=True,
+        color_axis="y",
+    ):
 
         # Need to better expose the options of which axis to use
         # Maybe have it as an extra key?
         x, y, z = vegm_array.shape
         if sand:
-            vegm_array[:, :, 2] = self._process_vegm('Sand', x, y, sand_axis)
+            vegm_array[:, :, 2] = self._process_vegm("Sand", x, y, sand_axis)
 
         if clay:
-            vegm_array[:, :, 3] = self._process_vegm('Clay', x, y, clay_axis)
+            vegm_array[:, :, 3] = self._process_vegm("Clay", x, y, clay_axis)
 
         if color:
-            vegm_array[:, :, 4] = self._process_vegm('Color', x, y,
-                                                     color_axis).astype(int)
+            vegm_array[:, :, 4] = self._process_vegm("Color", x, y, color_axis).astype(
+                int
+            )
 
-    def write_map(self, vegm_array=None, working_directory='.', dec_round=3):
+    def write_map(self, vegm_array=None, working_directory=".", dec_round=3):
         """Method to export drv_vegm.dat file based on keys or a 3D array of
            data
 
@@ -298,90 +305,96 @@ def write_map(self, vegm_array=None, working_directory='.', dec_round=3):
             - dec_round=3: sets the maximum decimal rounding for the lat, long,
               sand, and clay parameters.
         """
-        output_file = self._file(working_directory, 'map')
+        output_file = self._file(working_directory, "map")
 
         # Make sure we don't overwrite pre-existing data
-        self._ensure_writable(output_file, 'map')
+        self._ensure_writable(output_file, "map")
 
-        first_line = (' x  y  lat    lon    sand clay color  fractional '
-                      'coverage of grid by vegetation class (Must/Should Add '
-                      f'to 1.0) -- {self._auto_generated_file_string}')
-        second_line = '       (Deg)	 (Deg)  (%/100)   index'
+        first_line = (
+            " x  y  lat    lon    sand clay color  fractional "
+            "coverage of grid by vegetation class (Must/Should Add "
+            f"to 1.0) -- {self._auto_generated_file_string}"
+        )
+        second_line = "       (Deg)	 (Deg)  (%/100)   index"
 
-        land_col_map = {'column': 'land cover type'}
+        land_col_map = {"column": "land cover type"}
         land_covers = self._veg_params.LandNames
         if vegm_array is None:
             x = self.run.ComputationalGrid.NX
             y = self.run.ComputationalGrid.NY
             if x is None or y is None:
-                raise Exception('Computational grid has not been set')
+                raise Exception("Computational grid has not been set")
 
             vegm_array = np.zeros((x, y, 5))
             self._process_vegm_loc(vegm_array)
             self._process_vegm_soil(vegm_array)
             # CLM handles exactly 18 land cover types as a default
             if len(land_covers) > 18:
-                print(f'WARNING: CLM must be recompiled to accommodate '
-                      f'{len(land_covers)} land cover types.')
+                print(
+                    f"WARNING: CLM must be recompiled to accommodate "
+                    f"{len(land_covers)} land cover types."
+                )
             for name in land_covers:
                 vegm_array = np.dstack(
-                    (vegm_array, self._process_vegm(['LandFrac', name], x, y)))
+                    (vegm_array, self._process_vegm(["LandFrac", name], x, y))
+                )
 
-        with open(output_file, 'w') as wf:
-            wf.write(first_line + '\n')
+        with open(output_file, "w") as wf:
+            wf.write(first_line + "\n")
             if vegm_array.shape[2] < 23:
-                print(f'{len(land_covers)} land cover types specified. '
-                      f'Filling in zeros for {23 - vegm_array.shape[2]} '
-                      f'land cover types.')
-            wf.write(second_line + '\n')
+                print(
+                    f"{len(land_covers)} land cover types specified. "
+                    f"Filling in zeros for {23 - vegm_array.shape[2]} "
+                    f"land cover types."
+                )
+            wf.write(second_line + "\n")
             for i in range(vegm_array.shape[0]):
                 for j in range(vegm_array.shape[1]):
                     elements = [str(i + 1), str(j + 1)]
                     for k in range(max(vegm_array.shape[2], 23)):
                         if k == 4:
                             # dealing with color (needs to be int)
-                            elements.append(f'{int(vegm_array[i, j, k]):<7}')
+                            elements.append(f"{int(vegm_array[i, j, k]):<7}")
                         elif k < vegm_array.shape[2]:
-                            s = f'{round(vegm_array[i, j, k], dec_round):<7}'
+                            s = f"{round(vegm_array[i, j, k], dec_round):<7}"
                             elements.append(s)
                             if k > 4:
-                                land_col_map.update(
-                                    {k - 4: land_covers[k - 5]})
+                                land_col_map.update({k - 4: land_covers[k - 5]})
                         else:
-                            elements.append('0.0    ')
-                    wf.write('   ' + ' '.join(elements[:]) + '\n')
+                            elements.append("0.0    ")
+                    wf.write("   " + " ".join(elements[:]) + "\n")
 
-            print('Land cover column mapping')
+            print("Land cover column mapping")
             for key, value in land_col_map.items():
-                print(f'{key:<6}: {value}')
+                print(f"{key:<6}: {value}")
 
         return self
 
     def _land_cover_docs(self, clm_keys):
         # Get the help for the land cover without type info
         item = LandCoverParamItem()
-        path_to_item = self._veg_params_path + ['.{land_cover_name}']
+        path_to_item = self._veg_params_path + [".{land_cover_name}"]
         relative_start_ind = len(path_to_item)
         doc_strings = {}
         for key in clm_keys:
-            relative_path = '.'.join(CLM_KEY_DICT[key][relative_start_ind:])
-            doc = item.details(relative_path).get('help', '').strip()
+            relative_path = ".".join(CLM_KEY_DICT[key][relative_start_ind:])
+            doc = item.details(relative_path).get("help", "").strip()
             # Remove the type information from the front
-            doc = re.sub(TYPE_INFO_RE, '', doc)
+            doc = re.sub(TYPE_INFO_RE, "", doc)
             doc_strings[key] = doc
 
         return doc_strings
 
     @property
     def _land_cover_descriptions(self):
-        path = '/'.join(self._veg_params_path)
+        path = "/".join(self._veg_params_path)
         ret = {}
         for name in self._land_names:
-            desc = self.run.value(f'{path}/{name}/Description')
-            ret[name] = desc if desc else ''
+            desc = self.run.value(f"{path}/{name}/Description")
+            ret[name] = desc if desc else ""
         return ret
 
-    def write_parameters(self, vegp_data=None, working_directory='.'):
+    def write_parameters(self, vegp_data=None, working_directory="."):
         """Method to export drv_vegp.dat file based on dictionary of data
 
         Args:
@@ -390,10 +403,10 @@ def write_parameters(self, vegp_data=None, working_directory='.'):
             - working_directory='.': specifies where drv_vegp.dat
               file will be written
         """
-        output_file = self._file(working_directory, 'parameters')
+        output_file = self._file(working_directory, "parameters")
 
         # Make sure we don't overwrite pre-existing data
-        self._ensure_writable(output_file, 'parameters')
+        self._ensure_writable(output_file, "parameters")
 
         if vegp_data is None:
             # Extract the vegp data from the run object
@@ -407,24 +420,24 @@ def write_parameters(self, vegp_data=None, working_directory='.'):
         vegp_data = copy.deepcopy(vegp_data)
         self._resize_vegp_data(vegp_data)
 
-        output = f'! {self._auto_generated_file_string}\n'
+        output = f"! {self._auto_generated_file_string}\n"
         output += self._vegp_header
 
         # Make the header based upon the land names
         for i, name in enumerate(self._land_names, 1):
             description = descriptions[name]
-            output += f'! {i:>2} {name} {description}\n'
+            output += f"! {i:>2} {name} {description}\n"
 
         # Fill in not set values
         for i in range(len(self._land_names), self._min_num_land_covers):
-            output += f'! {i + 1:>2} not_set\n'
+            output += f"! {i + 1:>2} not_set\n"
 
-        output += '!' + '=' * 72 + '\n'
+        output += "!" + "=" * 72 + "\n"
 
         for key, val in vegp_data.items():
             doc = doc_strings[key]
-            output += '!\n'
-            output += f'{key:<15}{doc}\n'
+            output += "!\n"
+            output += f"{key:<15}{doc}\n"
             output += f'{" ".join(map(str, val))}\n'
 
         Path(output_file).write_text(output)
@@ -442,7 +455,7 @@ def _min_num_land_covers(self):
 
     @property
     def _veg_params_path(self):
-        return ['Solver', 'CLM', 'Vegetation', 'Parameters']
+        return ["Solver", "CLM", "Vegetation", "Parameters"]
 
     @property
     def _clm_solver(self):
@@ -463,11 +476,11 @@ def _land_names(self):
 
     @property
     def _vegp_header_file(self):
-        return Path(__file__).parent / 'ref/vegp_header.txt'
+        return Path(__file__).parent / "ref/vegp_header.txt"
 
     @property
     def _default_vegp_values_file(self):
-        return Path(__file__).parent / 'ref/default_vegp_values.json'
+        return Path(__file__).parent / "ref/default_vegp_values.json"
 
     @property
     def _vegp_header(self):
@@ -475,15 +488,15 @@ def _vegp_header(self):
 
     @property
     def _default_vegp_values(self):
-        if not hasattr(self, '_default_vegp_values_data'):
-            with open(self._default_vegp_values_file, 'r') as rf:
+        if not hasattr(self, "_default_vegp_values_data"):
+            with open(self._default_vegp_values_file, "r") as rf:
                 self._default_vegp_values_data = json.load(rf)
         return copy.deepcopy(self._default_vegp_values_data)
 
     def _generate_vegp_data(self):
         # Get the keys from the default values
         keys = list(self._default_vegp_values.keys())
-        land_items = self._veg_params.select('{LandCoverParamItem}')
+        land_items = self._veg_params.select("{LandCoverParamItem}")
 
         result = {}
         for key in keys:
@@ -495,11 +508,11 @@ def _generate_vegp_data(self):
     def _auto_generated_file_string(self):
         # This will be placed in the first line of every file that
         # was automatically generated by us.
-        return 'Automatically generated by pftools python'
+        return "Automatically generated by pftools python"
 
     def _contains_auto_generated_string(self, file_name):
         # Check if the file was automatically generated by us
-        with open(file_name, 'r') as rf:
+        with open(file_name, "r") as rf:
             return self._auto_generated_file_string in next(rf)
 
     def _writable(self, file_name, type):
@@ -520,25 +533,26 @@ def _writable(self, file_name, type):
     @property
     def _overwrite_settings_map(self):
         return {
-            'input': 'OverwriteDrvClmin',
-            'parameters': 'OverwriteDrvVegp',
-            'map': 'OverwriteDrvVegm',
+            "input": "OverwriteDrvClmin",
+            "parameters": "OverwriteDrvVegp",
+            "map": "OverwriteDrvVegm",
         }
 
     def _ensure_writable(self, file_name, type):
         # Make sure we can write to this file.
         if not self._writable(file_name, type):
             key = self._overwrite_settings_map[type]
-            msg = (f'{file_name} already exists, and {key} is false. '
-                   'Cannot overwrite.')
+            msg = (
+                f"{file_name} already exists, and {key} is false. " "Cannot overwrite."
+            )
             raise NotOverwritableException(msg)
 
     def _changes_detected(self, working_directory, type):
         # Check the history, and if there are changes, return True.
         items_to_check = {
-            'input': self._clm_solver.Input,
-            'map': self._clm_solver.Vegetation.Map,
-            'parameters': self._clm_solver.Vegetation.Parameters,
+            "input": self._clm_solver.Input,
+            "map": self._clm_solver.Vegetation.Map,
+            "parameters": self._clm_solver.Vegetation.Parameters,
         }
         if self._has_changes(items_to_check[type]):
             return True
@@ -552,9 +566,9 @@ def _changes_detected(self, working_directory, type):
     def _file(self, working_directory, type):
         # Get a full path to the file of the specified type
         file_map = {
-            'input': 'drv_clmin.dat',
-            'parameters': self._clm_solver.Input.File.VegTypeParameter,
-            'map': self._clm_solver.Input.File.VegTileSpecification,
+            "input": "drv_clmin.dat",
+            "parameters": self._clm_solver.Input.File.VegTypeParameter,
+            "map": self._clm_solver.Input.File.VegTileSpecification,
         }
         working_directory = get_absolute_path(working_directory)
         return Path(working_directory) / file_map[type]
@@ -563,28 +577,33 @@ def _print_not_written_warning(self, working_directory, type):
         # Print a warning that the file was not written.
         write_path = self._file(working_directory, type)
         key_name = self._overwrite_settings_map[type]
-        print(f'Warning: {write_path} will not be overwritten unless '
-              f'{key_name} is set to True. Changes to the file may not '
-              'be reflected in the calculation')
+        print(
+            f"Warning: {write_path} will not be overwritten unless "
+            f"{key_name} is set to True. Changes to the file may not "
+            "be reflected in the calculation"
+        )
 
     @property
     def _using_clm(self):
         # We can add other checks here if needed
-        return self.run.Solver.LSM == 'CLM'
+        return self.run.Solver.LSM == "CLM"
 
     @property
     def can_export(self):
         if self._using_clm:
-            land_param_items = self._veg_params.select('{LandCoverParamItem}')
-            land_map_items = self._veg_map.select('LandFrac/{LandFracCoverMapItem}')
-            return (land_param_items and land_map_items and
-                all(x is not None for x in land_param_items + land_map_items))
+            land_param_items = self._veg_params.select("{LandCoverParamItem}")
+            land_map_items = self._veg_map.select("LandFrac/{LandFracCoverMapItem}")
+            return (
+                land_param_items
+                and land_map_items
+                and all(x is not None for x in land_param_items + land_map_items)
+            )
 
         return False
 
     @property
     def _import_paths(self):
-        return self.run.__dict__.setdefault('_import_paths_', {})
+        return self.run.__dict__.setdefault("_import_paths_", {})
 
     @staticmethod
     def _has_changes(root):
@@ -597,9 +616,9 @@ def _has_changes(root):
             if not isinstance(item, PFDBObj):
                 continue
 
-            if hasattr(item, '_details_'):
+            if hasattr(item, "_details_"):
                 for details in item._details_.values():
-                    if details.get('history'):
+                    if details.get("history"):
                         return True
 
             if CLMExporter._has_changes(item):
diff --git a/pftools/python/parflow/tools/fs.py b/pftools/python/parflow/tools/fs.py
index 665afa3a5..eb9484896 100644
--- a/pftools/python/parflow/tools/fs.py
+++ b/pftools/python/parflow/tools/fs.py
@@ -13,6 +13,7 @@
 
 # -----------------------------------------------------------------------------
 
+
 def get_absolute_path(file_path):
     """Helper function to resolve a file path while using the proper
     working directory.
@@ -27,9 +28,9 @@ def get_absolute_path(file_path):
 
 # -----------------------------------------------------------------------------
 
-def cp(source, target_path='.'):
-    """Copying file/directory within python script
-    """
+
+def cp(source, target_path="."):
+    """Copying file/directory within python script"""
     full_source_path = get_absolute_path(source)
     full_target_path = get_absolute_path(target_path)
     try:
@@ -45,14 +46,14 @@ def cp(source, target_path='.'):
         print("Permission denied.")
     # For other errors
     except Exception:
-        print(f'Error occurred while copying {full_source_path}.')
+        print(f"Error occurred while copying {full_source_path}.")
 
 
 # -----------------------------------------------------------------------------
 
+
 def rm(path):
-    """Deleting file/directory within python script
-    """
+    """Deleting file/directory within python script"""
     full_path = Path(get_absolute_path(path))
     if full_path.exists():
         if full_path.is_dir():
@@ -63,9 +64,9 @@ def rm(path):
 
 # -----------------------------------------------------------------------------
 
+
 def mkdir(dir_name):
-    """mkdir within python script
-    """
+    """mkdir within python script"""
     full_path = Path(get_absolute_path(dir_name))
     if not full_path.exists():
         full_path.mkdir(parents=True)
@@ -73,22 +74,25 @@ def mkdir(dir_name):
 
 # -----------------------------------------------------------------------------
 
+
 def get_text_file_content(file_path):
     full_path = Path(get_absolute_path(file_path))
     if not full_path.exists():
-        raise Exception(f'{str(full_path)} does not exist!')
+        raise Exception(f"{str(full_path)} does not exist!")
 
     return full_path.read_text()
 
 
 # -----------------------------------------------------------------------------
 
+
 def exists(file_path):
     return Path(get_absolute_path(file_path)).exists()
 
 
 # -----------------------------------------------------------------------------
 
+
 def chdir(directory_path):
     full_path = get_absolute_path(directory_path)
     os.chdir(full_path)
diff --git a/pftools/python/parflow/tools/helper.py b/pftools/python/parflow/tools/helper.py
index 0648a69f5..b4cd4c8ff 100644
--- a/pftools/python/parflow/tools/helper.py
+++ b/pftools/python/parflow/tools/helper.py
@@ -13,6 +13,7 @@ def map_to_parent(pfdb_obj):
     """Helper function to extract the parent of a pfdb_obj"""
     return pfdb_obj._parent_
 
+
 # -----------------------------------------------------------------------------
 
 
@@ -20,14 +21,15 @@ def map_to_self(pfdb_obj):
     """Helper function to extract self of self (noop)"""
     return pfdb_obj
 
+
 # -----------------------------------------------------------------------------
 
 
 def map_to_child(name):
-    """Helper function that return a function for extracting a field name
-    """
+    """Helper function that return a function for extracting a field name"""
     return lambda pfdb_obj: pfdb_obj.__getitem__(name)
 
+
 # -----------------------------------------------------------------------------
 
 
@@ -37,24 +39,26 @@ def map_to_children_of_type(class_name):
     """
     return lambda pfdb_obj: pfdb_obj.get_children_of_type(class_name)
 
+
 # -----------------------------------------------------------------------------
 # Filter helpers
 # -----------------------------------------------------------------------------
 
+
 def filter_none(x):
     return x is not None
 
+
 # -----------------------------------------------------------------------------
 # Key dictionary helpers
 # -----------------------------------------------------------------------------
 
 
 def get_key_priority(key_name):
-    """Return number that can be used to sort keys in term of priority
-    """
+    """Return number that can be used to sort keys in term of priority"""
     priority_value = 0
-    path_token = key_name.split('.')
-    if 'Name' in key_name:
+    path_token = key_name.split(".")
+    if "Name" in key_name:
         priority_value -= 100
 
     for token in path_token:
@@ -68,6 +72,7 @@ def get_key_priority(key_name):
 
     return priority_value
 
+
 # -----------------------------------------------------------------------------
 
 # -----------------------------------------------------------------------------
@@ -76,32 +81,31 @@ def get_key_priority(key_name):
 
 
 def sort_dict(d, key=None):
-    """Create a key sorted dict
-    """
+    """Create a key sorted dict"""
     return {k: d[k] for k in sorted(d, key=key)}
 
+
 # -----------------------------------------------------------------------------
 
 
 def sort_dict_by_priority(d):
-    """Create a key sorted dict
-    """
+    """Create a key sorted dict"""
     return sort_dict(d, key=get_key_priority)
 
+
 # -----------------------------------------------------------------------------
 # Dictionary helpers
 # -----------------------------------------------------------------------------
 
 
 def get_or_create_dict(root, key_path, overriden_keys):
-    """Helper function to get/create a container dict for a given key path
-    """
+    """Helper function to get/create a container dict for a given key path"""
     current_container = root
     for i, key in enumerate(key_path):
         if key not in current_container:
             current_container[key] = {}
         elif not isinstance(current_container[key], dict):
-            overriden_keys['.'.join(key_path[:i+1])] = current_container[key]
+            overriden_keys[".".join(key_path[: i + 1])] = current_container[key]
             current_container[key] = {}
         current_container = current_container[key]
 
@@ -112,17 +116,18 @@ def get_or_create_dict(root, key_path, overriden_keys):
 # String helpers
 # -----------------------------------------------------------------------------
 
+
 def remove_prefix(s, prefix):
     if not s or not prefix or not s.startswith(prefix):
         return s
 
-    return s[len(prefix):]
+    return s[len(prefix) :]
 
 
 # First column is the regex. Second column is the replacement.
 INVALID_DOT_REGEX_SUBS = [
-    (re.compile(r'^\.([a-zA-Z]+)'), '\\1'),
-    (re.compile(r'([a-zA-Z]+)\.'), '\\1/')
+    (re.compile(r"^\.([a-zA-Z]+)"), "\\1"),
+    (re.compile(r"([a-zA-Z]+)\."), "\\1/"),
 ]
 
 
@@ -137,6 +142,7 @@ def _normalize_location(s):
 # Decorators
 # -----------------------------------------------------------------------------
 
+
 def normalize_location(func):
     """Assume the first string argument is location and normalize it.
 
@@ -145,6 +151,7 @@ def normalize_location(func):
         .Geom.Perm => Geom/Perm
 
     """
+
     @wraps(func)
     def wrapper(*args, **kwargs):
         args = list(args)
@@ -154,13 +161,16 @@ def wrapper(*args, **kwargs):
                 break
 
         return func(*args, **kwargs)
+
     return wrapper
 
 
 # -----------------------------------------------------------------------------
 
+
 def with_absolute_path(func):
     """Assume the first string argument is a path and resolve it."""
+
     @wraps(func)
     def wrapper(*args, **kwargs):
         args = list(args)
@@ -170,4 +180,5 @@ def wrapper(*args, **kwargs):
                 break
 
         return func(*args, **kwargs)
+
     return wrapper
diff --git a/pftools/python/parflow/tools/hydrology.py b/pftools/python/parflow/tools/hydrology.py
index 41110b7ca..1a103919b 100644
--- a/pftools/python/parflow/tools/hydrology.py
+++ b/pftools/python/parflow/tools/hydrology.py
@@ -4,11 +4,68 @@
 """
 
 import numpy as np
-
+import parflow.tools.io
 
 # -----------------------------------------------------------------------------
 
 
+def compute_hydraulic_head(pressure, z_0, dz):
+    """
+    Compute hydraulic head from a 3D pressure ndarray considering elevation.
+
+    This function uses the formula hh = hp + hz, where hh is the hydraulic head,
+    hp the pressure head, and hz the elevation head.
+
+    Args:
+        pressure (numpy.ndarray): 3D array of pressure values.
+        z_0 (float): Elevation of the top layer.
+        dz (float): Distance between z-layers.
+
+    Returns:
+        numpy.ndarray: 3D array of hydraulic head values.
+    """
+    num_layers = pressure.shape[0]
+    elevation = z_0 + np.arange(num_layers) * dz + dz / 2
+    hydraulic_head = pressure + elevation[:, np.newaxis, np.newaxis]
+    return hydraulic_head
+
+
+def compute_water_table_depth(saturation, top, dz):
+    """Computes the water table depth as the first cell with a saturation=1 starting from top.
+
+    Depth is depth below the top surface. Negative values indicate the water table was not found,
+    either below domain or the column at (i,j) is outside of domain.
+
+    Args:
+        saturation: ndarray of shape (nz, nx, ny)
+        top: ndarray of shape (1, nx, ny)
+        dz: distance between grid points in the z direction
+    Returns:
+        A new ndarray water_table_depth of shape (1, nx, ny) with depth values at each (i,j) location.
+    """
+    nz, nx, ny = saturation.shape
+    water_table_depth = np.ndarray((1, nx, ny))
+    for j in range(ny):
+        for i in range(nx):
+            top_k = top[0, i, j]
+            if top_k < 0:
+                # Inactive column so set to bogus value
+                water_table_depth[0, i, j] = -9999999.0
+            elif top_k < nz:
+                # Loop down until we find saturation >= 1
+                k = top_k
+                while k >= 0 and saturation[k, i, j] < 1:
+                    k -= 1
+
+                # Make sure water table was found in the column, set to bogus value if not
+                if k >= 0:
+                    water_table_depth[0, i, j] = (top_k - k) * dz
+                else:
+                    water_table_depth[0, i, j] = -9999999.0
+            else:
+                print(f"Error: Index in top (k={top_k}) is outside of domain (nz={nz})")
+
+
 def calculate_water_table_depth(pressure, saturation, dz):
     """
     Calculate water table depth from the land surface
@@ -30,7 +87,12 @@ def calculate_water_table_depth(pressure, saturation, dz):
     dz = np.hstack([dz, 0])
     # An unsaturated layer at the top
     # pad_width is a tuple of (n_before, n_after) for each dimension
-    saturation = np.pad(saturation, pad_width=((0, 1), (0, 0), (0, 0)), mode='constant', constant_values=0)
+    saturation = np.pad(
+        saturation,
+        pad_width=((0, 1), (0, 0), (0, 0)),
+        mode="constant",
+        constant_values=0,
+    )
 
     # Elevation of the center of each layer from the bottom (bottom layer to top layer)
     _elevation = np.cumsum(dz) - (dz / 2)
@@ -47,17 +109,22 @@ def calculate_water_table_depth(pressure, saturation, dz):
     we will not encounter this case.
     """
     z_indices = np.maximum(
-        np.argmax(saturation < 1, axis=0) - 1,  # find first unsaturated layer; back up one cell
-        0  # clamp min. index value to 0
+        np.argmax(saturation < 1, axis=0)
+        - 1,  # find first unsaturated layer; back up one cell
+        0,  # clamp min. index value to 0
     )
     # Make 3D with shape (1, ny, nx) to allow subsequent operations
     z_indices = z_indices[np.newaxis, ...]
 
-    saturation_elevation = np.take_along_axis(_elevation, z_indices, axis=0)  # shape (1, ny, nx)
+    saturation_elevation = np.take_along_axis(
+        _elevation, z_indices, axis=0
+    )  # shape (1, ny, nx)
     ponding_depth = np.take_along_axis(pressure, z_indices, axis=0)  # shape (1, ny, nx)
 
     wt_height = saturation_elevation + ponding_depth  # shape (1, ny, nx)
-    wt_height = np.clip(wt_height, 0, domain_thickness)  # water table height clamped between 0<->domain thickness
+    wt_height = np.clip(
+        wt_height, 0, domain_thickness
+    )  # water table height clamped between 0<->domain thickness
     wtd = domain_thickness - wt_height  # shape (1, ny, nx)
 
     return wtd.squeeze(axis=0)  # shape (ny, nx)
@@ -65,7 +132,10 @@ def calculate_water_table_depth(pressure, saturation, dz):
 
 # -----------------------------------------------------------------------------
 
-def calculate_subsurface_storage(porosity, pressure, saturation, specific_storage, dx, dy, dz, mask=None):
+
+def calculate_subsurface_storage(
+    porosity, pressure, saturation, specific_storage, dx, dy, dz, mask=None
+):
     """
     Calculate gridded subsurface storage across several layers.
 
@@ -90,7 +160,9 @@ def calculate_subsurface_storage(porosity, pressure, saturation, specific_storag
         mask = np.ones_like(porosity)
 
     mask = np.where(mask > 0, 1, 0)
-    dz = dz[:, np.newaxis, np.newaxis]  # make 3d so we can broadcast the multiplication below
+    dz = dz[
+        :, np.newaxis, np.newaxis
+    ]  # make 3d so we can broadcast the multiplication below
     incompressible = porosity * saturation * dz * dx * dy
     compressible = pressure * saturation * specific_storage * dz * dx * dy
     total = incompressible + compressible
@@ -100,6 +172,7 @@ def calculate_subsurface_storage(porosity, pressure, saturation, specific_storag
 
 # -----------------------------------------------------------------------------
 
+
 def calculate_surface_storage(pressure, dx, dy, mask=None):
     """
     Calculate gridded surface storage on the top layer.
@@ -121,12 +194,15 @@ def calculate_surface_storage(pressure, dx, dy, mask=None):
     surface_mask = mask[-1, ...]
     total = pressure[-1, ...] * dx * dy
     total[total < 0] = 0  # surface storage is 0 when pressure < 0
-    total[surface_mask == 0] = 0  # output values for points outside the mask are clamped to 0
+    total[surface_mask == 0] = (
+        0  # output values for points outside the mask are clamped to 0
+    )
     return total
 
 
 # -----------------------------------------------------------------------------
 
+
 def calculate_evapotranspiration(et, dx, dy, dz, mask=None):
     """
     Calculate gridded evapotranspiration across several layers.
@@ -143,7 +219,9 @@ def calculate_evapotranspiration(et, dx, dy, dz, mask=None):
         mask = np.ones_like(et)
 
     mask = np.where(mask > 0, 1, 0)
-    dz = dz[:, np.newaxis, np.newaxis]  # make 3d so we can broadcast the multiplication below
+    dz = dz[
+        :, np.newaxis, np.newaxis
+    ]  # make 3d so we can broadcast the multiplication below
     total = et * dz * dx * dy
     total[mask == 0] = 0  # output values for points outside the mask are clamped to 0
     return total
@@ -151,13 +229,18 @@ def calculate_evapotranspiration(et, dx, dy, dz, mask=None):
 
 # -----------------------------------------------------------------------------
 
+
 def _overland_flow(pressure_top, slopex, slopey, mannings, dx, dy):
     # Calculate fluxes across east and north faces
 
     # ---------------
     # The x direction
     # ---------------
-    qx = -(np.sign(slopex) * (np.abs(slopex) ** 0.5) / mannings) * (pressure_top ** (5 / 3)) * dy
+    qx = (
+        -(np.sign(slopex) * (np.abs(slopex) ** 0.5) / mannings)
+        * (pressure_top ** (5 / 3))
+        * dy
+    )
 
     # Upwinding to get flux across the east face of cells - based on qx[i] if it is positive and qx[i+1] if negative
     qeast = np.maximum(0, qx[:, :-1]) - np.maximum(0, -qx[:, 1:])
@@ -173,7 +256,11 @@ def _overland_flow(pressure_top, slopex, slopey, mannings, dx, dy):
     # ---------------
     # The y direction
     # ---------------
-    qy = -(np.sign(slopey) * (np.abs(slopey) ** 0.5) / mannings) * (pressure_top ** (5 / 3)) * dx
+    qy = (
+        -(np.sign(slopey) * (np.abs(slopey) ** 0.5) / mannings)
+        * (pressure_top ** (5 / 3))
+        * dx
+    )
 
     # Upwinding to get flux across the north face of cells - based in qy[j] if it is positive and qy[j+1] if negative
     qnorth = np.maximum(0, qy[:-1, :]) - np.maximum(0, -qy[1:, :])
@@ -191,7 +278,10 @@ def _overland_flow(pressure_top, slopex, slopey, mannings, dx, dy):
 
 # -----------------------------------------------------------------------------
 
-def _overland_flow_kinematic(mask, pressure_top, slopex, slopey, mannings, dx, dy, epsilon):
+
+def _overland_flow_kinematic(
+    mask, pressure_top, slopex, slopey, mannings, dx, dy, epsilon
+):
     ##### --- ######
     #     qx       #
     ##### --- ######
@@ -211,28 +301,43 @@ def _overland_flow_kinematic(mask, pressure_top, slopex, slopey, mannings, dx, d
     # and copy the slopex, slopey and mannings values from the '1' cells to the corresponding '0' cells
     _x, _y = np.where(np.diff(mask, axis=1, append=0) == 1)
     slopex[(_x, _y)] = slopex[(_x, _y + 1)]
-    slopey[(_x, _y)] = slopey[(_x, _y + 1)] 
-    mannings[(_x, _y)] = mannings[(_x, _y + 1)] 
-    
+    slopey[(_x, _y)] = slopey[(_x, _y + 1)]
+    mannings[(_x, _y)] = mannings[(_x, _y + 1)]
+
     slope = np.maximum(epsilon, np.hypot(slopex, slopey))
-    
+
     # Upwind pressure - this is for the north and east face of all cells
     # The slopes are calculated across these boundaries so the upper x boundaries are included in these
     # calculations. The lower x boundaries are added further down as q_x0
-    pressure_top_padded = np.pad(pressure_top[:, 1:], ((0, 0,), (0, 1)))  # pad right
-    pupwindx = np.maximum(0, np.sign(slopex) * pressure_top_padded) + np.maximum(0, -np.sign(slopex) * pressure_top)
-    
+    pressure_top_padded = np.pad(
+        pressure_top[:, 1:],
+        (
+            (
+                0,
+                0,
+            ),
+            (0, 1),
+        ),
+    )  # pad right
+    pupwindx = np.maximum(0, np.sign(slopex) * pressure_top_padded) + np.maximum(
+        0, -np.sign(slopex) * pressure_top
+    )
+
     flux_factor = np.sqrt(slope) * mannings
-    
+
     # Flux across the x directions
     q_x = -slopex / flux_factor * pupwindx ** (5 / 3) * dy
-    
+
     # Fix the lower x boundary
     # Use the slopes of the first column
-    q_x0 = -slopex[:, 0] / flux_factor[:, 0] * np.maximum(0, np.sign(slopex[:, 0]) * pressure_top[:, 0]) ** (5 / 3) * dy
+    q_x0 = (
+        -slopex[:, 0]
+        / flux_factor[:, 0]
+        * np.maximum(0, np.sign(slopex[:, 0]) * pressure_top[:, 0]) ** (5 / 3)
+        * dy
+    )
     qeast = np.hstack([q_x0[:, np.newaxis], q_x])
 
-
     ##### --- ######
     #   qy   #
     ##### --- ######
@@ -246,15 +351,25 @@ def _overland_flow_kinematic(mask, pressure_top, slopex, slopey, mannings, dx, d
     slopey[(_x, _y)] = slopey[(_x + 1, _y)]
     slopex[(_x, _y)] = slopex[(_x + 1, _y)]
     mannings[(_x, _y)] = mannings[(_x + 1, _y)]
-    
+
     slope = np.maximum(epsilon, np.hypot(slopex, slopey))
-    
 
     # Upwind pressure - this is for the north and east face of all cells
     # The slopes are calculated across these boundaries so the upper y boundaries are included in these
     # calculations. The lower y boundaries are added further down as q_y0
-    pressure_top_padded = np.pad(pressure_top[1:, :], ((0, 1,), (0, 0)))  # pad bottom
-    pupwindy = np.maximum(0, np.sign(slopey) * pressure_top_padded) + np.maximum(0, -np.sign(slopey) * pressure_top)
+    pressure_top_padded = np.pad(
+        pressure_top[1:, :],
+        (
+            (
+                0,
+                1,
+            ),
+            (0, 0),
+        ),
+    )  # pad bottom
+    pupwindy = np.maximum(0, np.sign(slopey) * pressure_top_padded) + np.maximum(
+        0, -np.sign(slopey) * pressure_top
+    )
 
     flux_factor = np.sqrt(slope) * mannings
 
@@ -263,15 +378,31 @@ def _overland_flow_kinematic(mask, pressure_top, slopex, slopey, mannings, dx, d
 
     # Fix the lower y boundary
     # Use the slopes of the first row
-    q_y0 = -slopey[0, :] / flux_factor[0, :] * np.maximum(0, np.sign(slopey[0, :]) * pressure_top[0, :]) ** (5 / 3) * dx
+    q_y0 = (
+        -slopey[0, :]
+        / flux_factor[0, :]
+        * np.maximum(0, np.sign(slopey[0, :]) * pressure_top[0, :]) ** (5 / 3)
+        * dx
+    )
     qnorth = np.vstack([q_y0, q_y])
-    
+
     return qeast, qnorth
 
+
 # -----------------------------------------------------------------------------
 
-def calculate_overland_fluxes(pressure, slopex, slopey, mannings, dx, dy, flow_method='OverlandKinematic', epsilon=1e-5,
-                              mask=None):
+
+def calculate_overland_fluxes(
+    pressure,
+    slopex,
+    slopey,
+    mannings,
+    dx,
+    dy,
+    flow_method="OverlandKinematic",
+    epsilon=1e-5,
+    mask=None,
+):
     """
     Calculate overland fluxes across grid faces
 
@@ -319,26 +450,30 @@ def calculate_overland_fluxes(pressure, slopex, slopey, mannings, dx, dy, flow_m
 
     # Handle cases when expected 2D arrays come in as 3D
     if slopex.ndim == 3:
-        assert slopex.shape[0] == 1, f'Expected shape[0] of 3D ndarray {slopex} to be 1'
+        assert slopex.shape[0] == 1, f"Expected shape[0] of 3D ndarray {slopex} to be 1"
         slopex = slopex.squeeze(axis=0)
     if slopey.ndim == 3:
-        assert slopey.shape[0] == 1, f'Expected shape[0] of 3D ndarray {slopey} to be 1'
+        assert slopey.shape[0] == 1, f"Expected shape[0] of 3D ndarray {slopey} to be 1"
         slopey = slopey.squeeze(axis=0)
     mannings = np.array(mannings)
     if mannings.ndim == 3:
-        assert mannings.shape[0] == 1, f'Expected shape[0] of 3D ndarray {mannings} to be 1'
+        assert (
+            mannings.shape[0] == 1
+        ), f"Expected shape[0] of 3D ndarray {mannings} to be 1"
         mannings = mannings.squeeze(axis=0)
 
     pressure_top = pressure[-1, ...].copy()
     pressure_top = np.nan_to_num(pressure_top)
     pressure_top[pressure_top < 0] = 0
 
-    assert flow_method in ('OverlandFlow', 'OverlandKinematic'), 'Unknown flow method'
-    if flow_method == 'OverlandKinematic':
+    assert flow_method in ("OverlandFlow", "OverlandKinematic"), "Unknown flow method"
+    if flow_method == "OverlandKinematic":
         if mask is None:
             mask = np.ones_like(pressure)
         mask = np.where(mask > 0, 1, 0)
-        qeast, qnorth = _overland_flow_kinematic(mask, pressure_top, slopex, slopey, mannings, dx, dy, epsilon)
+        qeast, qnorth = _overland_flow_kinematic(
+            mask, pressure_top, slopex, slopey, mannings, dx, dy, epsilon
+        )
     else:
         qeast, qnorth = _overland_flow(pressure_top, slopex, slopey, mannings, dx, dy)
 
@@ -347,8 +482,18 @@ def calculate_overland_fluxes(pressure, slopex, slopey, mannings, dx, dy, flow_m
 
 # -----------------------------------------------------------------------------
 
-def calculate_overland_flow_grid(pressure, slopex, slopey, mannings, dx, dy, flow_method='OverlandKinematic',
-                                 epsilon=1e-5, mask=None):
+
+def calculate_overland_flow_grid(
+    pressure,
+    slopex,
+    slopey,
+    mannings,
+    dx,
+    dy,
+    flow_method="OverlandKinematic",
+    epsilon=1e-5,
+    mask=None,
+):
     """
     Calculate overland outflow per grid cell of a domain
 
@@ -367,12 +512,25 @@ def calculate_overland_flow_grid(pressure, slopex, slopey, mannings, dx, dy, flo
     :return: A ny-by-nx ndarray of overland flow values
     """
     mask = np.where(mask > 0, 1, 0)
-    qeast, qnorth = calculate_overland_fluxes(pressure, slopex, slopey, mannings, dx, dy, flow_method=flow_method,
-                                              epsilon=epsilon, mask=mask)
+    qeast, qnorth = calculate_overland_fluxes(
+        pressure,
+        slopex,
+        slopey,
+        mannings,
+        dx,
+        dy,
+        flow_method=flow_method,
+        epsilon=epsilon,
+        mask=mask,
+    )
 
     # Outflow is a positive qeast[i,j+1] or qnorth[i+1,j] or a negative qeast[i,j], qnorth[i,j]
-    outflow = np.maximum(0, qeast[:, 1:]) + np.maximum(0, -qeast[:, :-1]) + \
-              np.maximum(0, qnorth[1:, :]) + np.maximum(0, -qnorth[:-1, :])
+    outflow = (
+        np.maximum(0, qeast[:, 1:])
+        + np.maximum(0, -qeast[:, :-1])
+        + np.maximum(0, qnorth[1:, :])
+        + np.maximum(0, -qnorth[:-1, :])
+    )
 
     # Set the outflow values outside the mask to 0
     outflow[mask[-1, ...] == 0] = 0
@@ -382,8 +540,18 @@ def calculate_overland_flow_grid(pressure, slopex, slopey, mannings, dx, dy, flo
 
 # -----------------------------------------------------------------------------
 
-def calculate_overland_flow(pressure, slopex, slopey, mannings, dx, dy, flow_method='OverlandKinematic',
-                            epsilon=1e-5, mask=None):
+
+def calculate_overland_flow(
+    pressure,
+    slopex,
+    slopey,
+    mannings,
+    dx,
+    dy,
+    flow_method="OverlandKinematic",
+    epsilon=1e-5,
+    mask=None,
+):
     """
     Calculate overland outflow out of a domain
 
@@ -399,10 +567,19 @@ def calculate_overland_flow(pressure, slopex, slopey, mannings, dx, dy, flow_met
         This is set using the Solver.OverlandKinematic.Epsilon key in Parflow.
     :param mask: A nz-by-ny-by-nx ndarray of mask values (bottom layer to top layer)
         If None, assumed to be an nz-by-ny-by-nx ndarray of 1s.
-    :return: A ny-by-nx ndarray of overland flow values
+    :return: A float value representing the total overland flow over the domain.
     """
-    qeast, qnorth = calculate_overland_fluxes(pressure, slopex, slopey, mannings, dx, dy, flow_method=flow_method,
-                                              epsilon=epsilon, mask=mask)
+    qeast, qnorth = calculate_overland_fluxes(
+        pressure,
+        slopex,
+        slopey,
+        mannings,
+        dx,
+        dy,
+        flow_method=flow_method,
+        epsilon=epsilon,
+        mask=mask,
+    )
 
     if mask is not None:
         mask = np.where(mask > 0, 1, 0)
@@ -411,8 +588,7 @@ def calculate_overland_flow(pressure, slopex, slopey, mannings, dx, dy, flow_met
         surface_mask = np.ones_like(slopex)  # shape ny, nx
 
     # Important to typecast mask to float to avoid values wrapping around when performing a np.diff
-    surface_mask = surface_mask.astype('float')
-
+    surface_mask = surface_mask.astype("float")
     # Find edge pixels for our surface mask along each face - N/S/W/E
     # All of these have shape (ny, nx) and values as 0/1
 
@@ -426,13 +602,17 @@ def calculate_overland_flow(pressure, slopex, slopey, mannings, dx, dy, flow_met
     edge_east = np.maximum(0, -np.diff(surface_mask, axis=1, append=0))
 
     # North flux is the sum of +ve qnorth values (shifted up by one) on north edges
-    flux_north = np.sum(np.maximum(0, np.roll(qnorth, -1, axis=0)[np.where(edge_north == 1)]))
+    flux_north = np.sum(
+        np.maximum(0, np.roll(qnorth, -1, axis=0)[np.where(edge_north == 1)])
+    )
     # South flux is the negated sum of -ve qnorth values for south edges
     flux_south = np.sum(np.maximum(0, -qnorth[np.where(edge_south == 1)]))
     # West flux is the negated sum of -ve qeast values of west edges
     flux_west = np.sum(np.maximum(0, -qeast[np.where(edge_west == 1)]))
     # East flux is the sum of +ve qeast values (shifted left by one) for east edges
-    flux_east = np.sum(np.maximum(0, np.roll(qeast, -1, axis=1)[np.where(edge_east == 1)]))
+    flux_east = np.sum(
+        np.maximum(0, np.roll(qeast, -1, axis=1)[np.where(edge_east == 1)])
+    )
 
     flux = flux_north + flux_south + flux_west + flux_east
     return flux
diff --git a/pftools/python/parflow/tools/io.py b/pftools/python/parflow/tools/io.py
index f0435f4b3..bd80c5747 100644
--- a/pftools/python/parflow/tools/io.py
+++ b/pftools/python/parflow/tools/io.py
@@ -22,16 +22,20 @@
 import itertools
 import json
 from pathlib import Path
+
 try:
     from numba import jit, njit
 except ImportError:
     # Some systems may not have numba capabilities
     def jit(*args, **kwargs):
         """Dummy decorator, does nothing"""
+
         def _decorator(func):
             return func
+
         return _decorator
 
+
 from numbers import Number
 import pandas as pd
 import numpy as np
@@ -56,8 +60,41 @@ def _decorator(func):
     from yaml import Dumper as YAMLDumper
 
 
-def read_pfb(file: str, keys: dict=None, mode: str='full', z_first: bool=True,
-             read_sg_info: bool=True):
+def read_pfsb(file_path):
+    """
+    Read a ParFlow scattered binary file (pfsb) and return the data.
+
+    The data will be returned as a numpy array of shape (nz, ny, nx).
+    To avoid confusion, there is no option to toggle 'z_first' on and
+    off.
+    """
+    with open(file_path, "rb") as f:
+        # Read the header
+        x, y, z = struct.unpack(">ddd", f.read(24))
+        nx, ny, nz = struct.unpack(">iii", f.read(12))
+        dx, dy, dz = struct.unpack(">ddd", f.read(24))
+        num_subgrids = struct.unpack(">i", f.read(4))[0]
+        data = np.zeros((nz, ny, nx), dtype=np.float64)
+
+        for _ in range(num_subgrids):
+            # Skip most of subgrid header
+            f.seek(36, 1)
+            num_nonzero_data = struct.unpack(">i", f.read(4))[0]
+
+            for idx in range(num_nonzero_data):
+                i, j, k = struct.unpack(">iii", f.read(12))
+                data[k, j, i] = struct.unpack(">d", f.read(8))[0]
+
+    return data
+
+
+def read_pfb(
+    file: str,
+    keys: dict = None,
+    mode: str = "full",
+    z_first: bool = True,
+    read_sg_info: bool = True,
+):
     """
     Read a single pfb file, and return the data therein
 
@@ -78,7 +115,7 @@ def read_pfb(file: str, keys: dict=None, mode: str='full', z_first: bool=True,
         for more information about what modes are available.
     :param read_sg_info:
         Precalculating subgrid information does not always work correctly for
-        some files, especially velocity files. If ``True``, read subgrid info 
+        some files, especially velocity files. If ``True``, read subgrid info
         directly from the pfb file (slower, but more robust)
     :return:
         An nd array containing the data from the pfb file.
@@ -88,29 +125,39 @@ def read_pfb(file: str, keys: dict=None, mode: str='full', z_first: bool=True,
             data = pfb.read_all_subgrids(mode=mode, z_first=z_first)
         else:
             base_header = pfb.header
-            start_x = keys.get('x', {}).get('start', None) or 0
-            start_y = keys.get('y', {}).get('start', None) or 0
-            start_z = keys.get('z', {}).get('start', None) or 0
-            stop_x =  keys.get('x', {}).get('stop', None) or base_header['nx']
-            stop_y =  keys.get('y', {}).get('stop', None) or base_header['ny']
-            stop_z =  keys.get('z', {}).get('stop', None) or base_header['nz']
+            start_x = keys.get("x", {}).get("start", None) or 0
+            start_y = keys.get("y", {}).get("start", None) or 0
+            start_z = keys.get("z", {}).get("start", None) or 0
+            stop_x = keys.get("x", {}).get("stop", None) or base_header["nx"]
+            stop_y = keys.get("y", {}).get("stop", None) or base_header["ny"]
+            stop_z = keys.get("z", {}).get("stop", None) or base_header["nz"]
             nx = np.max([stop_x - start_x, 1])
             ny = np.max([stop_y - start_y, 1])
             nz = np.max([stop_z - start_z, 1])
             data = pfb.read_subarray(
-                        start_x, start_y, start_z, nx, ny, nz, z_first=z_first)
+                start_x, start_y, start_z, nx, ny, nz, z_first=z_first
+            )
     return data
 
 
 # -----------------------------------------------------------------------------
 
+
 def write_pfb(
-    file, array,
-    p=1, q=1, r=1,
-    x=0.0, y=0.0, z=0.0,
-    dx=1.0, dy=1.0, dz=1.0,
-    z_first=True, dist=True,
-    **kwargs
+    file,
+    array,
+    p=1,
+    q=1,
+    r=1,
+    x=0.0,
+    y=0.0,
+    z=0.0,
+    dx=1.0,
+    dy=1.0,
+    dz=1.0,
+    z_first=True,
+    dist=True,
+    **kwargs,
 ):
     """
     Write a single pfb file. The data must be a 3D numpy array with float64
@@ -155,12 +202,14 @@ def write_pfb(
         args by passing in a dictionary with `**dict`.
     """
     if array.dtype != np.float64:
-        raise ValueError(f"Arrays written to pfb must be of type np.float64!"
-                          + " Found {array.dtype} instead!")
+        raise ValueError(
+            f"Arrays written to pfb must be of type np.float64!"
+            + " Found {array.dtype} instead!"
+        )
 
     if len(array.shape) == 3:
         if z_first:
-                nz, ny, nx = array.shape
+            nz, ny, nx = array.shape
         else:
             nx, ny, nz = array.shape
     elif len(array.shape) == 2:
@@ -176,44 +225,44 @@ def write_pfb(
         nx, ny, nz, p, q, r
     )
 
-    with open(file, 'wb') as f:
+    with open(file, "wb") as f:
         # Write the file header
-        f.write(struct.pack('>d', float(x)))
-        f.write(struct.pack('>d', float(y)))
-        f.write(struct.pack('>d', float(z)))
-        f.write(struct.pack('>i', int(nx)))
-        f.write(struct.pack('>i', int(ny)))
-        f.write(struct.pack('>i', int(nz)))
-        f.write(struct.pack('>d', float(dx)))
-        f.write(struct.pack('>d', float(dy)))
-        f.write(struct.pack('>d', float(dz)))
-        f.write(struct.pack('>i', int(n_subgrids)))
+        f.write(struct.pack(">d", float(x)))
+        f.write(struct.pack(">d", float(y)))
+        f.write(struct.pack(">d", float(z)))
+        f.write(struct.pack(">i", int(nx)))
+        f.write(struct.pack(">i", int(ny)))
+        f.write(struct.pack(">i", int(nz)))
+        f.write(struct.pack(">d", float(dx)))
+        f.write(struct.pack(">d", float(dy)))
+        f.write(struct.pack(">d", float(dz)))
+        f.write(struct.pack(">i", int(n_subgrids)))
 
         # loop over subgrids:
         for off, loc, start, shape in zip(sg_offs, sg_locs, sg_starts, sg_shapes):
             # Write the subgrid header
-            for sgh in itertools.chain(start, shape, [1,1,1]):
-                f.write(struct.pack('>i', int(sgh)))
+            for sgh in itertools.chain(start, shape, [1, 1, 1]):
+                f.write(struct.pack(">i", int(sgh)))
 
             mm = np.memmap(
                 file,
                 dtype=np.float64,
-                mode='r+',
+                mode="r+",
                 offset=off,
                 shape=tuple(shape),
-                order='F'
+                order="F",
             )
             # Gather shapes & extents
             s_ix, s_iy, s_iz = start
             n_ix, n_iy, n_iz = shape
             if z_first:
-                mm[:] = array[s_iz:s_iz+n_iz,
-                              s_iy:s_iy+n_iy,
-                              s_ix:s_ix+n_ix].T.byteswap()
+                mm[:] = array[
+                    s_iz : s_iz + n_iz, s_iy : s_iy + n_iy, s_ix : s_ix + n_ix
+                ].T.byteswap()
             else:
-                mm[:] = array[s_ix:s_ix+n_ix,
-                              s_iy:s_iy+n_iy,
-                              s_iz:s_iz+n_iz].byteswap()
+                mm[:] = array[
+                    s_ix : s_ix + n_ix, s_iy : s_iy + n_iy, s_iz : s_iz + n_iz
+                ].byteswap()
             # Seek to offset + the size of the subgrid
             f.seek(off + 8 * np.prod(shape))
 
@@ -235,17 +284,18 @@ def write_dist(file, sg_offs):
         for i, s in enumerate(sg_offs):
             # Need to account for header bytes
             real_off = s - 100 if i == 0 else s - 36
-            dist_fp.write(f'{real_off}\n')
+            dist_fp.write(f"{real_off}\n")
 
 
 # -----------------------------------------------------------------------------
 
+
 def read_pfb_sequence(
     file_seq: Iterable[str],
     keys=None,
-    z_first: bool=True,
-    z_is: str='z',
-    read_sg_info: bool=False
+    z_first: bool = True,
+    z_is: str = "z",
+    read_sg_info: bool = False,
 ):
     """
     An efficient wrapper to read a sequence of pfb files. This
@@ -273,9 +323,9 @@ def read_pfb_sequence(
         'z', 'time', 'variable'. Default is 'z'.
     :param read_sg_info:
         Precalculating subgrid information does not always work correctly for
-        some files, especially velocity files. If ``True``, read subgrid info 
+        some files, especially velocity files. If ``True``, read subgrid info
         directly from the pfb file (slower, but more robust)
-    
+
     :return:
         An nd array containing the data from the files.
     """
@@ -290,14 +340,14 @@ def read_pfb_sequence(
         base_sg_chunks = pfb_init.chunks
         base_sg_coords = pfb_init.coords
     if not keys:
-        nx, ny, nz = base_header['nx'], base_header['ny'], base_header['nz']
+        nx, ny, nz = base_header["nx"], base_header["ny"], base_header["nz"]
     else:
-        start_x = keys.get('x', {}).get('start', None) or 0
-        start_y = keys.get('y', {}).get('start', None) or 0
-        start_z = keys.get(z_is, {}).get('start', None) or 0
-        stop_x =  keys.get('x', {}).get('stop', None) or base_header['nx']
-        stop_y =  keys.get('y', {}).get('stop', None) or base_header['ny']
-        stop_z =  keys.get(z_is, {}).get('stop', None) or base_header['nz']
+        start_x = keys.get("x", {}).get("start", None) or 0
+        start_y = keys.get("y", {}).get("start", None) or 0
+        start_z = keys.get(z_is, {}).get("start", None) or 0
+        stop_x = keys.get("x", {}).get("stop", None) or base_header["nx"]
+        stop_y = keys.get("y", {}).get("stop", None) or base_header["ny"]
+        stop_z = keys.get(z_is, {}).get("stop", None) or base_header["nz"]
         nx = np.max([stop_x - start_x, 1])
         ny = np.max([stop_y - start_y, 1])
         nz = np.max([stop_z - start_z, 1])
@@ -319,12 +369,13 @@ def read_pfb_sequence(
             pfb.coords = base_sg_coords
             pfb.chunks = base_sg_chunks
             if not keys:
-                subseq_data = pfb.read_all_subgrids(mode='full', z_first=z_first)
+                subseq_data = pfb.read_all_subgrids(mode="full", z_first=z_first)
             else:
                 subseq_data = pfb.read_subarray(
-                        start_x, start_y, start_z, nx, ny, nz, z_first=z_first)
-            pfb_seq[i, :, : ,:] = subseq_data
-    if z_is == 'time':
+                    start_x, start_y, start_z, nx, ny, nz, z_first=z_first
+                )
+            pfb_seq[i, :, :, :] = subseq_data
+    if z_is == "time":
         if z_first:
             pfb_seq = np.concatenate(pfb_seq, axis=0)
         else:
@@ -334,6 +385,7 @@ def read_pfb_sequence(
 
 # -----------------------------------------------------------------------------
 
+
 class ParflowBinaryReader:
     """
     The ParflowBinaryReader, unsurprisingly, provides functionality
@@ -373,51 +425,36 @@ class ParflowBinaryReader:
     :param read_sg_info:
         Whether or not to read subgrid information directly from the pfb file.
         Precalculating subgrid information does not always work correctly for
-        some files, especially velocity files. This reads the subgrid offset 
-        bytes, subgrid indices and subgrid shapes, then computes the subgrid 
+        some files, especially velocity files. This reads the subgrid offset
+        bytes, subgrid indices and subgrid shapes, then computes the subgrid
         coordinates subgrid locations, and chunk sizes as normal
-        
+
     """
 
     def __init__(
         self,
         file: str,
-        precompute_subgrid_info: bool=True,
-        p: int=None,
-        q: int=None,
-        r: int=None,
-        header: Mapping[str, Number]=None,
-        read_sg_info: bool=False
+        precompute_subgrid_info: bool = True,
+        p: int = None,
+        q: int = None,
+        r: int = None,
+        header: Mapping[str, Number] = None,
+        read_sg_info: bool = False,
     ):
         self.filename = file
-        self.f = open(self.filename, 'rb')
+        self.f = open(self.filename, "rb")
         if not header:
             self.header = self.read_header()
         else:
             self.header = header
 
         if np.all([p, q, r]):
-            self.header['p'] = p
-            self.header['q'] = q
-            self.header['r'] = r
-
-        if not ('p' in self.header
-            and 'q' in self.header
-            and 'r' in self.header):
-            # If p, q, and r aren't given we can precompute them
-            # NOTE: This is a bit of a fallback and may not always work
-            eps = 1 - 1e-6
-            first_sg_head = self.read_subgrid_header()
-            self.header['p'] = int((self.header['nx'] / first_sg_head['nx']) + eps)
-            self.header['q'] = int((self.header['ny'] / first_sg_head['ny']) + eps)
-            self.header['r'] = int((self.header['nz'] / first_sg_head['nz']) + eps)
-
-        if precompute_subgrid_info:
-            self.compute_subgrid_info()
-
-        if read_sg_info:
-            self.read_subgrid_info()
-            
+            self.header["p"] = p
+            self.header["q"] = q
+            self.header["r"] = r
+
+        self.read_subgrid_info()
+
     def close(self):
         self.f.close()
 
@@ -428,15 +465,15 @@ def __exit__(self, type, value, traceback):
         self.f.close()
 
     def compute_subgrid_info(self):
-        """ Computes the subgrid information """
+        """Computes the subgrid information"""
         try:
             sg_offs, sg_locs, sg_starts, sg_shapes = precalculate_subgrid_info(
-                self.header['nx'],
-                self.header['ny'],
-                self.header['nz'],
-                self.header['p'],
-                self.header['q'],
-                self.header['r']
+                self.header["nx"],
+                self.header["ny"],
+                self.header["nz"],
+                self.header["p"],
+                self.header["q"],
+                self.header["r"],
             )
         except:
             raise ValueError(self.header)
@@ -448,33 +485,60 @@ def compute_subgrid_info(self):
         self.coords = self._compute_coords()
 
     def read_subgrid_info(self):
-        """ Read the header for each subgrid directly from the pfb file, rather 
-        than calculating it via precalculate_subgrid_info """
-        
+        """Read the header for each subgrid directly from the pfb file, rather
+        than calculating it via precalculate_subgrid_info"""
+
         sg_shapes = []
         sg_offs = []
         sg_locs = []
         sg_starts = []
 
         off = 64
-        for sg_num in range(self.header['n_subgrids']):
+        for sg_num in range(self.header["n_subgrids"]):
             # Read and move past the current subgrid header
             sg_head = self.read_subgrid_header(off)
-            off += 36 
-            
-            sg_starts.append([sg_head['ix'], sg_head['iy'], sg_head['iz']])
-            sg_shapes.append([sg_head['nx'], sg_head['ny'], sg_head['nz']])
+            off += 36
+
+            sg_starts.append([sg_head["ix"], sg_head["iy"], sg_head["iz"]])
+            sg_shapes.append([sg_head["nx"], sg_head["ny"], sg_head["nz"]])
             sg_offs.append(off)
 
+            # Finally, move past the current subgrid before next iteration
+            off += sg_head["sg_size"] * 8
+
+        # Calculate p, q, r from subgrid shapes
+        p, q, r = 0, 0, 0
+        x, y, z = 0, 0, 0
+
+        for shape in sg_shapes:
+            if x == self.header["nx"]:
+                break
+            p = p + 1
+            x = x + shape[0]
+
+        for shape in sg_shapes[::p]:
+            if y == self.header["ny"]:
+                break
+            q = q + 1
+            y = y + shape[1]
+
+        for shape in sg_shapes[:: p * q]:
+            if z == self.header["nz"]:
+                break
+            r = r + 1
+            z = z + shape[2]
+
+        self.header["p"] = p
+        self.header["q"] = q
+        self.header["r"] = r
+
+        for sg_num in range(self.header["n_subgrids"]):
             # Calculate subgrid locs instead of reading from file
-            sg_p, sg_q, sg_r = get_subgrid_loc(sg_num, self.header['p'], 
-                                                       self.header['q'], 
-                                                       self.header['r'])
+            sg_p, sg_q, sg_r = get_subgrid_loc(
+                sg_num, self.header["p"], self.header["q"], self.header["r"]
+            )
             sg_locs.append((sg_p, sg_q, sg_r))
-            
-            # Finally, move past the current subgrid before next iteration
-            off += sg_head['sg_size']*8
-        
+
         self.subgrid_offsets = np.array(sg_offs)
         self.subgrid_locations = np.array(sg_locs)
         self.subgrid_start_indices = np.array(sg_starts)
@@ -494,11 +558,15 @@ def _compute_chunks(self) -> Mapping[str, tuple]:
              'y': tuple_with_len_q,
              'z': tuple_with_len_r}
         """
-        p, q, r = self.header['p'], self.header['q'], self.header['r'],
-        x_chunks = tuple(self.subgrid_shapes[:,0][0:p].flatten())
-        y_chunks = tuple(self.subgrid_shapes[:,1][0:p*q:p].flatten())
-        z_chunks = tuple(self.subgrid_shapes[:,2][0:p*q*r:p*q].flatten())
-        return {'x': x_chunks, 'y': y_chunks, 'z': z_chunks}
+        p, q, r = (
+            self.header["p"],
+            self.header["q"],
+            self.header["r"],
+        )
+        x_chunks = tuple(self.subgrid_shapes[:, 0][0:p].flatten())
+        y_chunks = tuple(self.subgrid_shapes[:, 1][0 : p * q : p].flatten())
+        z_chunks = tuple(self.subgrid_shapes[:, 2][0 : p * q * r : p * q].flatten())
+        return {"x": x_chunks, "y": y_chunks, "z": z_chunks}
 
     def _compute_coords(self) -> Mapping[str, Iterable[Iterable[int]]]:
         """
@@ -518,8 +586,8 @@ def _compute_coords(self) -> Mapping[str, Iterable[Iterable[int]]]:
                    (n1+1, n1+2, ... n1+n2),
                     ... for ni in self.chunks['z']],
         """
-        coords = {'x': [], 'y': [], 'z': []}
-        for c in ['x', 'y', 'z']:
+        coords = {"x": [], "y": [], "z": []}
+        for c in ["x", "y", "z"]:
             chunk_start = 0
             for chunk in self.chunks[c]:
                 coords[c].append(np.arange(chunk_start, chunk_start + chunk))
@@ -530,51 +598,55 @@ def read_header(self):
         """Reads the header"""
         self.f.seek(0)
         header = {}
-        header['x'] = struct.unpack('>d', self.f.read(8))[0]
-        header['y'] = struct.unpack('>d', self.f.read(8))[0]
-        header['z'] = struct.unpack('>d', self.f.read(8))[0]
-        header['nx'] = struct.unpack('>i', self.f.read(4))[0]
-        header['ny'] = struct.unpack('>i', self.f.read(4))[0]
-        header['nz'] = struct.unpack('>i', self.f.read(4))[0]
-        header['dx'] = struct.unpack('>d', self.f.read(8))[0]
-        header['dy'] = struct.unpack('>d', self.f.read(8))[0]
-        header['dz'] = struct.unpack('>d', self.f.read(8))[0]
-        header['n_subgrids'] = struct.unpack('>i', self.f.read(4))[0]
+        header["x"] = struct.unpack(">d", self.f.read(8))[0]
+        header["y"] = struct.unpack(">d", self.f.read(8))[0]
+        header["z"] = struct.unpack(">d", self.f.read(8))[0]
+        header["nx"] = struct.unpack(">i", self.f.read(4))[0]
+        header["ny"] = struct.unpack(">i", self.f.read(4))[0]
+        header["nz"] = struct.unpack(">i", self.f.read(4))[0]
+        header["dx"] = struct.unpack(">d", self.f.read(8))[0]
+        header["dy"] = struct.unpack(">d", self.f.read(8))[0]
+        header["dz"] = struct.unpack(">d", self.f.read(8))[0]
+        header["n_subgrids"] = struct.unpack(">i", self.f.read(4))[0]
         return header
 
-    def read_subgrid_header(self, skip_bytes: int=64):
+    def read_subgrid_header(self, skip_bytes: int = 64):
         """Reads a subgrid header at the position ``skip_bytes``"""
         self.f.seek(skip_bytes)
         sg_header = {}
         header_len = 9
 
         # Apologies, this is kind of ugly, but faster than using struct
-        (sg_header['ix'],
-         sg_header['iy'],
-         sg_header['iz'],
-         sg_header['nx'],
-         sg_header['ny'],
-         sg_header['nz'],
-         sg_header['rx'],
-         sg_header['ry'],
-         sg_header['rz']) = np.memmap(self.f,
-                                      dtype=np.int32,
-                                      offset=skip_bytes,
-                                      mode='r',
-                                      shape=(header_len,),
-                                      order='F').byteswap()
-        sg_header['sg_size'] = np.prod([sg_header[n] for n in ['nx', 'ny', 'nz']])
+        (
+            sg_header["ix"],
+            sg_header["iy"],
+            sg_header["iz"],
+            sg_header["nx"],
+            sg_header["ny"],
+            sg_header["nz"],
+            sg_header["rx"],
+            sg_header["ry"],
+            sg_header["rz"],
+        ) = np.memmap(
+            self.f,
+            dtype=np.int32,
+            offset=skip_bytes,
+            mode="r",
+            shape=(header_len,),
+            order="F",
+        ).byteswap()
+        sg_header["sg_size"] = np.prod([sg_header[n] for n in ["nx", "ny", "nz"]])
         return sg_header
 
     def read_subarray(
-            self,
-            start_x: int,
-            start_y: int,
-            start_z: int=0,
-            nx: int=1,
-            ny: int=1,
-            nz: int=None,
-            z_first: bool=True
+        self,
+        start_x: int,
+        start_y: int,
+        start_z: int = 0,
+        nx: int = 1,
+        ny: int = 1,
+        nz: int = None,
+        z_first: bool = True,
     ) -> np.ndarray:
         """
         Read a subsection of the full pfb file. For an example of what happens
@@ -618,8 +690,9 @@ def read_subarray(
         :returns:
             A nd array with shape (nx, ny, nz).
         """
+
         def _get_final_clip(start, end, coords):
-            """ Helper to clean up code at the end of this """
+            """Helper to clean up code at the end of this"""
             x0 = np.flatnonzero(start == coords)
             x0 = 0 if not x0 else x0[0]
             x1 = np.flatnonzero(end == coords)
@@ -627,12 +700,14 @@ def _get_final_clip(start, end, coords):
             return slice(x0, x1)
 
         def _get_needed_subgrids(start, end, coords):
-            """ Helper function to clean up subgrid selection """
+            """Helper function to clean up subgrid selection"""
             for s, c in enumerate(coords):
-                if start in c: break
+                if start in c:
+                    break
             for e, c in enumerate(coords):
-                if end in c: break
-            return np.arange(s, e+1)
+                if end in c:
+                    break
+            return np.arange(s, e + 1)
 
         if not start_x:
             start_x = 0
@@ -641,20 +716,20 @@ def _get_needed_subgrids(start, end, coords):
         if not start_z:
             start_z = 0
         if not nx:
-            nx = self.header['nx']
+            nx = self.header["nx"]
         if not ny:
-            ny = self.header['ny']
+            ny = self.header["ny"]
         if not nz:
-            nz = self.header['nz']
+            nz = self.header["nz"]
 
         end_x = start_x + nx
         end_y = start_y + ny
         end_z = start_z + nz
-        p, q, r = self.header['p'], self.header['q'], self.header['r']
+        p, q, r = self.header["p"], self.header["q"], self.header["r"]
         # Convert to numpy array for simpler indexing
-        x_coords = np.array(self.coords['x'], dtype=object)
-        y_coords = np.array(self.coords['y'], dtype=object)
-        z_coords = np.array(self.coords['z'], dtype=object)
+        x_coords = np.array(self.coords["x"], dtype=object)
+        y_coords = np.array(self.coords["y"], dtype=object)
+        z_coords = np.array(self.coords["z"], dtype=object)
 
         # Determine which subgrids we need to read
         p_subgrids = _get_needed_subgrids(start_x, end_x, x_coords)
@@ -673,13 +748,13 @@ def _get_needed_subgrids(start, end, coords):
         full_size = (len(x_sg_coords), len(y_sg_coords), len(z_sg_coords))
         bounding_data = np.empty(full_size, dtype=np.float64)
         subgrid_iter = itertools.product(p_subgrids, q_subgrids, r_subgrids)
-        for (xsg, ysg, zsg) in subgrid_iter:
+        for xsg, ysg, zsg in subgrid_iter:
             subgrid_idx = xsg + (p * ysg) + (p * q * zsg)
             # Set up the indices to insert subgrid data into the bounding data
             x0, y0, z0 = self.subgrid_start_indices[subgrid_idx]
             x0, y0, z0 = x0 - x_min, y0 - y_min, z0 - z_min
             dx, dy, dz = self.subgrid_shapes[subgrid_idx]
-            x1, y1, z1 = x0 + dx, y0 + dy, z0+ dz
+            x1, y1, z1 = x0 + dx, y0 + dy, z0 + dz
             bounding_data[x0:x1, y0:y1, z0:z1] = self.iloc_subgrid(subgrid_idx)
 
         # Now clip out the exact part from the bounding box
@@ -692,7 +767,6 @@ def _get_needed_subgrids(start, end, coords):
             ret_data = bounding_data[clip_x, clip_y, clip_z]
         return ret_data
 
-
     def loc_subgrid(self, sg_p: int, sg_q: int, sg_r: int) -> np.ndarray:
         """
         Read a subgrid given it's (sg_p, sg_q, sg_r) coordinate in the subgrid-grid.
@@ -706,7 +780,7 @@ def loc_subgrid(self, sg_p: int, sg_q: int, sg_r: int) -> np.ndarray:
         :returns:
             The data from the (sg_p, sg_q, sg_r)'th subgrid.
         """
-        p, q, r = self.header['p'], self.header['q'], self.header['r']
+        p, q, r = self.header["p"], self.header["q"], self.header["r"]
         subgrid_idx = sg_p + (p * sg_q) + (q * p * sg_r)
         return self.iloc_subgrid(subgrid_idx)
 
@@ -723,9 +797,7 @@ def iloc_subgrid(self, idx: int) -> np.ndarray:
         shape = self.subgrid_shapes[idx]
         return self._backend_iloc_subgrid(offset, shape)
 
-    def _backend_iloc_subgrid(
-            self, offset: int, shape: Iterable[int]
-    ) -> np.ndarray:
+    def _backend_iloc_subgrid(self, offset: int, shape: Iterable[int]) -> np.ndarray:
         """
         Backend function for memory mapping data from the pfb file on disk.
 
@@ -739,16 +811,16 @@ def _backend_iloc_subgrid(
         mm = np.memmap(
             self.f,
             dtype=np.float64,
-            mode='r',
+            mode="r",
             offset=offset,
             shape=tuple(shape),
-            order='F'
+            order="F",
         ).byteswap()
         data = np.array(mm)
         return data
 
     def read_all_subgrids(
-            self, mode: str='full', z_first: bool=True
+        self, mode: str = "full", z_first: bool = True
     ) -> Union[Iterable[np.ndarray], np.ndarray]:
         """
         Read all of the subgrids in the file.
@@ -768,41 +840,46 @@ def read_all_subgrids(
             also will be numpy array of floats with dimensions (sg_nx, sg_ny, sg_nz) where
             each of sg_nx, sg_ny, and sg_nz are the size of the subgrid array.
         """
-        if mode not in ['flat', 'tiled', 'full']:
-            raise Exception('mode must be one of flat, tiled, or full')
-        if mode in ['flat', 'tiled']:
+        if mode not in ["flat", "tiled", "full"]:
+            raise Exception("mode must be one of flat, tiled, or full")
+        if mode in ["flat", "tiled"]:
             all_data = []
-            for i in range(self.header['n_subgrids']):
+            for i in range(self.header["n_subgrids"]):
                 if z_first:
                     all_data.append(self.iloc_subgrid(i).T)
                 else:
                     all_data.append(self.iloc_subgrid(i))
-            if mode == 'tiled':
+            if mode == "tiled":
                 if z_first:
-                    tiled_shape = tuple(self.header[dim] for dim in ['r', 'q', 'p'])
+                    tiled_shape = tuple(self.header[dim] for dim in ["r", "q", "p"])
                     all_data = np.array(all_data, dtype=object).reshape(tiled_shape)
                 else:
-                    tiled_shape = tuple(self.header[dim] for dim in ['p', 'q', 'r'])
+                    tiled_shape = tuple(self.header[dim] for dim in ["p", "q", "r"])
                     all_data = np.array(all_data, dtype=object).reshape(tiled_shape)
-        elif mode == 'full':
+        elif mode == "full":
             if z_first:
-                full_shape = tuple(self.header[dim] for dim in ['nz', 'ny', 'nx'])
+                full_shape = tuple(self.header[dim] for dim in ["nz", "ny", "nx"])
             else:
-                full_shape = tuple(self.header[dim] for dim in ['nx', 'ny', 'nz'])
-            chunks = self.chunks['x'], self.chunks['y'], self.chunks['z']
+                full_shape = tuple(self.header[dim] for dim in ["nx", "ny", "nz"])
+            chunks = self.chunks["x"], self.chunks["y"], self.chunks["z"]
             all_data = np.empty(full_shape, dtype=np.float64)
-            for i in range(self.header['n_subgrids']):
+            for i in range(self.header["n_subgrids"]):
                 nx, ny, nz = self.subgrid_shapes[i]
                 ix, iy, iz = self.subgrid_start_indices[i]
                 if z_first:
-                    all_data[iz:iz+nz, iy:iy+ny, ix:ix+nx] = self.iloc_subgrid(i).T
+                    all_data[iz : iz + nz, iy : iy + ny, ix : ix + nx] = (
+                        self.iloc_subgrid(i).T
+                    )
                 else:
-                    all_data[ix:ix+nx, iy:iy+ny, iz:iz+nz] = self.iloc_subgrid(i)
+                    all_data[ix : ix + nx, iy : iy + ny, iz : iz + nz] = (
+                        self.iloc_subgrid(i)
+                    )
         return all_data
 
 
 # -----------------------------------------------------------------------------
 
+
 @jit(nopython=True)
 def get_maingrid_and_remainder(nx, ny, nz, p, q, r):
     """
@@ -827,14 +904,15 @@ def get_maingrid_and_remainder(nx, ny, nz, p, q, r):
     mg_nx = int(nx / p)
     mg_ny = int(ny / q)
     mg_nz = int(nz / r)
-    rm_nx = (nx % p)
-    rm_ny = (ny % q)
-    rm_nz = (nz % r)
+    rm_nx = nx % p
+    rm_ny = ny % q
+    rm_nz = nz % r
     return mg_nx, mg_ny, mg_nz, rm_nx, rm_ny, rm_nz
 
 
 # -----------------------------------------------------------------------------
 
+
 @jit(nopython=True)
 def get_subgrid_loc(sel_subgrid, p, q, r):
     """
@@ -850,7 +928,7 @@ def get_subgrid_loc(sel_subgrid, p, q, r):
         The number of subgrids along the z axis.
     """
     sg_r = int(np.floor(sel_subgrid / (p * q)))
-    sg_q = int(np.floor((sel_subgrid - (sg_r*p*q)) / p))
+    sg_q = int(np.floor((sel_subgrid - (sg_r * p * q)) / p))
     sg_p = int(sel_subgrid - sg_r * (p * q) - (sg_q * p))
     subgrid_loc = (sg_p, sg_q, sg_r)
     return subgrid_loc
@@ -858,12 +936,9 @@ def get_subgrid_loc(sel_subgrid, p, q, r):
 
 # -----------------------------------------------------------------------------
 
+
 @jit(nopython=True)
-def subgrid_lower_left(
-    mg_nx, mg_ny, mg_nz,
-    sg_p, sg_q, sg_r,
-    rm_nx, rm_ny, rm_nz
-):
+def subgrid_lower_left(mg_nx, mg_ny, mg_nz, sg_p, sg_q, sg_r, rm_nx, rm_ny, rm_nz):
     """
     Get the index of the lower left corner of a subgrid.
 
@@ -894,12 +969,9 @@ def subgrid_lower_left(
 
 # -----------------------------------------------------------------------------
 
+
 @jit(nopython=True)
-def subgrid_size(
-    mg_nx, mg_ny, mg_nz,
-    sg_p, sg_q, sg_r,
-    rm_nx, rm_ny, rm_nz
-):
+def subgrid_size(mg_nx, mg_ny, mg_nz, sg_p, sg_q, sg_r, rm_nx, rm_ny, rm_nz):
     """
     Get the size of a subgrid
 
@@ -922,14 +994,15 @@ def subgrid_size(
     :param rm_nz:
         Remainder from the maingrid calculation on the z-axis
     """
-    sg_nx = mg_nx if sg_p >= rm_nx else mg_nx+1
-    sg_ny = mg_ny if sg_q >= rm_ny else mg_ny+1
-    sg_nz = mg_nz if sg_r >= rm_nz else mg_nz+1
+    sg_nx = mg_nx if sg_p >= rm_nx else mg_nx + 1
+    sg_ny = mg_ny if sg_q >= rm_ny else mg_ny + 1
+    sg_nz = mg_nz if sg_r >= rm_nz else mg_nz + 1
     return sg_nx, sg_ny, sg_nz
 
 
 # -----------------------------------------------------------------------------
 
+
 @jit(nopython=True)
 def precalculate_subgrid_info(nx, ny, nz, p, q, r):
     """
@@ -967,25 +1040,22 @@ def precalculate_subgrid_info(nx, ny, nz, p, q, r):
     off = 64
     for sg_num in range(n_subgrids):
         # Move past the current header and previous subgrid
-        off += 36 +  (8 * (sg_nx * sg_ny * sg_nz))
+        off += 36 + (8 * (sg_nx * sg_ny * sg_nz))
         subgrid_offsets.append(off)
 
-        (mg_nx, mg_ny, mg_nz,
-         rm_nx, rm_ny, rm_nz) = get_maingrid_and_remainder(nx, ny, nz, p, q, r)
+        (mg_nx, mg_ny, mg_nz, rm_nx, rm_ny, rm_nz) = get_maingrid_and_remainder(
+            nx, ny, nz, p, q, r
+        )
         sg_p, sg_q, sg_r = get_subgrid_loc(sg_num, p, q, r)
         subgrid_locs.append((sg_p, sg_q, sg_r))
 
         ix, iy, iz = subgrid_lower_left(
-            mg_nx, mg_ny, mg_nz,
-            sg_p, sg_q, sg_r,
-            rm_nx, rm_ny, rm_nz
+            mg_nx, mg_ny, mg_nz, sg_p, sg_q, sg_r, rm_nx, rm_ny, rm_nz
         )
         subgrid_begin_idxs.append((ix, iy, iz))
 
         sg_nx, sg_ny, sg_nz = subgrid_size(
-            mg_nx, mg_ny, mg_nz,
-            sg_p, sg_q, sg_r,
-            rm_nx, rm_ny, rm_nz
+            mg_nx, mg_ny, mg_nz, sg_p, sg_q, sg_r, rm_nx, rm_ny, rm_nz
         )
         subgrid_shapes.append((sg_nx, sg_ny, sg_nz))
     return subgrid_offsets, subgrid_locs, subgrid_begin_idxs, subgrid_shapes
@@ -993,8 +1063,8 @@ def precalculate_subgrid_info(nx, ny, nz, p, q, r):
 
 # -----------------------------------------------------------------------------
 
-def load_patch_matrix_from_image_file(file_name, color_to_patch=None,
-                                      fall_back_id=0):
+
+def load_patch_matrix_from_image_file(file_name, color_to_patch=None, fall_back_id=0):
     import imageio
 
     im = imageio.imread(file_name)
@@ -1012,15 +1082,15 @@ def load_patch_matrix_from_image_file(file_name, color_to_patch=None,
         colors = []
 
         def _to_key(c, num):
-            return ','.join([f'{c[i]}' for i in range(num)])
+            return ",".join([f"{c[i]}" for i in range(num)])
 
         to_key_1 = partial(_to_key, num=1)
         to_key_2 = partial(_to_key, num=2)
         to_key_3 = partial(_to_key, num=3)
 
         for key, value in color_to_patch.items():
-            hex_color = key.lstrip('#')
-            color = tuple(int(hex_color[i:i + 2], 16) for i in (0, 2, 4))
+            hex_color = key.lstrip("#")
+            color = tuple(int(hex_color[i : i + 2], 16) for i in (0, 2, 4))
             colors.append((color, value))
             size1.add(to_key_1(color))
             size2.add(to_key_2(color))
@@ -1034,11 +1104,13 @@ def _to_key(c, num):
         if len(colors) == len(size1):
             to_key = to_key_1
 
-        print(f'Sizes: colors({len(colors)}), 1({len(size1)}), '
-              f'2({len(size2)}), 3({len(size3)})')
+        print(
+            f"Sizes: colors({len(colors)}), 1({len(size1)}), "
+            f"2({len(size2)}), 3({len(size3)})"
+        )
 
         if to_key is None:
-            raise Exception('You have duplicate colors')
+            raise Exception("You have duplicate colors")
 
         fast_map = {}
         for color_patch in colors:
@@ -1057,6 +1129,7 @@ def _to_key(c, num):
 
 # -----------------------------------------------------------------------------
 
+
 def load_patch_matrix_from_asc_file(file_name):
     ncols = -1
     nrows = -1
@@ -1070,9 +1143,9 @@ def load_patch_matrix_from_asc_file(file_name):
                 in_header = False
             except Exception:
                 key, value = line.split()
-                if key == 'ncols':
+                if key == "ncols":
                     ncols = int(value)
-                if key == 'nrows':
+                if key == "nrows":
                     nrows = int(value)
                 nb_line_to_skip += 1
 
@@ -1084,6 +1157,7 @@ def load_patch_matrix_from_asc_file(file_name):
 
 # -----------------------------------------------------------------------------
 
+
 def load_patch_matrix_from_sa_file(file_name):
     i_size = -1
     j_size = -1
@@ -1098,67 +1172,77 @@ def load_patch_matrix_from_sa_file(file_name):
 
 # -----------------------------------------------------------------------------
 
-def write_patch_matrix_as_asc(matrix, file_name, xllcorner=0.0, yllcorner=0.0,
-                              cellsize=1.0, NODATA_value=0, **kwargs):
+
+def write_patch_matrix_as_asc(
+    matrix,
+    file_name,
+    xllcorner=0.0,
+    yllcorner=0.0,
+    cellsize=1.0,
+    NODATA_value=0,
+    **kwargs,
+):
     """Write asc for pfsol"""
     height, width = matrix.shape
-    with open(file_name, 'w') as out:
-        out.write(f'ncols          {width}\n')
-        out.write(f'nrows          {height}\n')
-        out.write(f'xllcorner      {xllcorner}\n')
-        out.write(f'yllcorner      {yllcorner}\n')
-        out.write(f'cellsize       {cellsize}\n')
-        out.write(f'NODATA_value   {NODATA_value}\n')
+    with open(file_name, "w") as out:
+        out.write(f"ncols          {width}\n")
+        out.write(f"nrows          {height}\n")
+        out.write(f"xllcorner      {xllcorner}\n")
+        out.write(f"yllcorner      {yllcorner}\n")
+        out.write(f"cellsize       {cellsize}\n")
+        out.write(f"NODATA_value   {NODATA_value}\n")
         # asc are vertically flipped
         for j in range(height):
             for i in range(width):
-                out.write(f'{matrix[height - j - 1, i]}\n')
+                out.write(f"{matrix[height - j - 1, i]}\n")
 
 
 # -----------------------------------------------------------------------------
 
+
 def write_patch_matrix_as_sa(matrix, file_name, **kwargs):
     """Write asc for pfsol"""
     nrows, ncols = matrix.shape
-    with open(file_name, 'w') as out:
-        out.write(f'{ncols} {nrows} 1\n')
+    with open(file_name, "w") as out:
+        out.write(f"{ncols} {nrows} 1\n")
         it = np.nditer(matrix)
         for value in it:
-            out.write(f'{value}\n')
+            out.write(f"{value}\n")
 
 
 # -----------------------------------------------------------------------------
 
+
 def write_dict_as_pfidb(dict_obj, file_name):
-    """Write a Python dict in a pfidb format inside the provided file_name
-    """
-    with open(file_name, 'w') as out:
-        out.write(f'{len(dict_obj)}\n')
+    """Write a Python dict in a pfidb format inside the provided file_name"""
+    with open(file_name, "w") as out:
+        out.write(f"{len(dict_obj)}\n")
         for key in dict_obj:
-            out.write(f'{len(key)}\n')
-            out.write(f'{key}\n')
+            out.write(f"{len(key)}\n")
+            out.write(f"{key}\n")
             value = dict_obj[key]
-            out.write(f'{len(str(value))}\n')
-            out.write(f'{str(value)}\n')
+            out.write(f"{len(str(value))}\n")
+            out.write(f"{str(value)}\n")
 
 
 # -----------------------------------------------------------------------------
 
+
 def write_dict_as_yaml(dict_obj, file_name):
-    """Write a Python dict in a pfidb format inside the provided file_name
-    """
+    """Write a Python dict in a pfidb format inside the provided file_name"""
     yaml_obj = {}
     overriden_keys = {}
     for key, value in dict_obj.items():
-        keys_path = key.split('.')
-        get_or_create_dict(
-            yaml_obj, keys_path[:-1], overriden_keys)[keys_path[-1]] = value
+        keys_path = key.split(".")
+        get_or_create_dict(yaml_obj, keys_path[:-1], overriden_keys)[
+            keys_path[-1]
+        ] = value
 
     # Push value back to yaml
     for key, value in overriden_keys.items():
-        keys_path = key.split('.')
+        keys_path = key.split(".")
         value_obj = get_or_create_dict(yaml_obj, keys_path, {})
-        value_obj['_value_'] = value
+        value_obj["_value_"] = value
 
     output = yaml.dump(sort_dict(yaml_obj), Dumper=YAMLDumper)
     Path(file_name).write_text(output)
@@ -1166,14 +1250,15 @@ def write_dict_as_yaml(dict_obj, file_name):
 
 # -----------------------------------------------------------------------------
 
+
 def write_dict_as_json(dict_obj, file_name):
-    """Write a Python dict in a json format inside the provided file_name
-    """
+    """Write a Python dict in a json format inside the provided file_name"""
     Path(file_name).write_text(json.dumps(dict_obj, indent=2))
 
 
 # -----------------------------------------------------------------------------
 
+
 def write_dict(dict_obj, file_name):
     """Write a Python dict into a file_name using the extension to
     determine its format.
@@ -1182,18 +1267,19 @@ def write_dict(dict_obj, file_name):
     sorted_dict = sort_dict(dict_obj)
 
     ext = Path(file_name).suffix[1:].lower()
-    if ext in ['yaml', 'yml']:
+    if ext in ["yaml", "yml"]:
         write_dict_as_yaml(sorted_dict, file_name)
-    elif ext == 'pfidb':
+    elif ext == "pfidb":
         write_dict_as_pfidb(sorted_dict, file_name)
-    elif ext == 'json':
+    elif ext == "json":
         write_dict_as_json(sorted_dict, file_name)
     else:
-        raise Exception(f'Could not find writer for {file_name}')
+        raise Exception(f"Could not find writer for {file_name}")
 
 
 # -----------------------------------------------------------------------------
 
+
 def to_native_type(string):
     """Converting a string to a value in native format.
     Used for converting .pfidb files
@@ -1207,51 +1293,51 @@ def to_native_type(string):
 
     # Handle boolean type
     lower_str = string.lower()
-    if lower_str in ['true', 'false']:
-        return lower_str[0] == 't'
+    if lower_str in ["true", "false"]:
+        return lower_str[0] == "t"
 
     return string
 
 
 # -----------------------------------------------------------------------------
 
+
 def read_pfidb(file_path):
-    """Load pfidb file into a Python dict
-    """
+    """Load pfidb file into a Python dict"""
     result_dict = {}
-    action = 'nb_lines'  # nb_lines, size, string
+    action = "nb_lines"  # nb_lines, size, string
     size = 0
-    key = ''
-    value = ''
+    key = ""
+    value = ""
     string_type_count = 0
     full_path = get_absolute_path(file_path)
 
-    with open(full_path, 'r') as input_file:
+    with open(full_path, "r") as input_file:
         for line in input_file:
-            if action == 'string':
+            if action == "string":
                 if string_type_count % 2 == 0:
                     key = line[:size]
                 else:
                     value = line[:size]
                     result_dict[key] = to_native_type(value)
                 string_type_count += 1
-                action = 'size'
+                action = "size"
 
-            elif action == 'size':
+            elif action == "size":
                 size = int(line)
-                action = 'string'
+                action = "string"
 
-            elif action == 'nb_lines':
-                action = 'size'
+            elif action == "nb_lines":
+                action = "size"
 
     return result_dict
 
 
 # -----------------------------------------------------------------------------
 
+
 def read_yaml(file_path):
-    """Load yaml file into a Python dict
-    """
+    """Load yaml file into a Python dict"""
     path = Path(file_path)
     if not path.exists():
         return {}
@@ -1261,17 +1347,18 @@ def read_yaml(file_path):
 
 # -----------------------------------------------------------------------------
 
+
 def _read_clmin(file_name):
     """function to load in drv_clmin.dat files
 
-       Args:
-           - file_name: name of drv_clmin.dat file
+    Args:
+        - file_name: name of drv_clmin.dat file
 
-       Returns:
-           dictionary of key/value pairs of variables in file
+    Returns:
+        dictionary of key/value pairs of variables in file
     """
     clm_vars = {}
-    with open(file_name, 'r') as rf:
+    with open(file_name, "r") as rf:
         for line in rf:
             # skip if first 15 are empty or exclamation
             if line and line[0].islower():
@@ -1286,23 +1373,24 @@ def _read_clmin(file_name):
 
 # -----------------------------------------------------------------------------
 
+
 def _read_vegm(file_name):
     """function to load in drv_vegm.dat files
 
-       Args:
-           - file_name: name of drv_vegm.dat file
+    Args:
+        - file_name: name of drv_vegm.dat file
 
-       Returns:
-           3D numpy array for domain, with 3rd dimension defining each column
-           in the vegm.dat file except for x/y
+    Returns:
+        3D numpy array for domain, with 3rd dimension defining each column
+        in the vegm.dat file except for x/y
     """
     # Assume first two lines are comments and use generic column names
     df = pd.read_csv(file_name, delim_whitespace=True, skiprows=2, header=None)
-    df.columns = [f'c{i}' for i in range(df.shape[1])]
+    df.columns = [f"c{i}" for i in range(df.shape[1])]
 
     # Number of columns and rows determined by last line of file
-    nx = int(df.iloc[-1]['c0'])
-    ny = int(df.iloc[-1]['c1'])
+    nx = int(df.iloc[-1]["c0"])
+    ny = int(df.iloc[-1]["c1"])
     # Don't use 'x' and 'y' columns
     feature_cols = df.columns[2:]
     # Stack everything into (ny, nx, n_features)
@@ -1314,21 +1402,22 @@ def _read_vegm(file_name):
 
 # -----------------------------------------------------------------------------
 
+
 def _read_vegp(file_name):
     """function to load in drv_vegp.dat files
 
-       Args:
-           - file_name: name of drv_vegp.dat file
+    Args:
+        - file_name: name of drv_vegp.dat file
 
-       Returns:
-           Dictionary with keys as variables and values as lists of parameter
-           values for each of the 18 land cover types
+    Returns:
+        Dictionary with keys as variables and values as lists of parameter
+        values for each of the 18 land cover types
     """
     vegp_data = {}
     current_var = None
-    with open(file_name, 'r') as rf:
+    with open(file_name, "r") as rf:
         for line in rf:
-            if not line or line[0] == '!':
+            if not line or line[0] == "!":
                 continue
 
             split = line.split()
@@ -1343,15 +1432,12 @@ def _read_vegp(file_name):
 
 # -----------------------------------------------------------------------------
 
-def read_clm(file_name, type='clmin'):
-    type_map = {
-        'clmin': _read_clmin,
-        'vegm': _read_vegm,
-        'vegp': _read_vegp
-    }
+
+def read_clm(file_name, type="clmin"):
+    type_map = {"clmin": _read_clmin, "vegm": _read_vegm, "vegp": _read_vegp}
 
     if type not in type_map:
-        raise Exception(f'Unknown clm type: {type}')
+        raise Exception(f"Unknown clm type: {type}")
 
     return type_map[type](get_absolute_path(file_name))
 
@@ -1392,7 +1478,7 @@ def time(self):
     @time.setter
     def time(self, t):
         self._time = int(t)
-        self._ts = f'{self._time:0>{self._t_padding}}'
+        self._ts = f"{self._time:0>{self._t_padding}}"
 
     @property
     def times(self):
@@ -1454,7 +1540,7 @@ def shape(self):
         return (
             self._run.ComputationalGrid.NZ,
             self._run.ComputationalGrid.NY,
-            self._run.ComputationalGrid.NX
+            self._run.ComputationalGrid.NX,
         )
 
     @property
@@ -1468,10 +1554,10 @@ def dy(self):
     @property
     def dz(self):
         if self._run.Solver.Nonlinear.VariableDz:
-            assert self._run.dzScale.Type == 'nzList'
+            assert self._run.dzScale.Type == "nzList"
             dz_scale = []
             for i in range(self._run.dzScale.nzListNumber):
-                dz_scale.append(self._run.Cell[str(i)]['dzScale']['Value'])
+                dz_scale.append(self._run.Cell[str(i)]["dzScale"]["Value"])
             dz_scale = np.array(dz_scale)
         else:
             dz_scale = np.ones((self._run.ComputationalGrid.NZ,))
@@ -1485,7 +1571,7 @@ def dz(self):
 
     @property
     def mannings(self):
-        return self._pfb_to_array(f'{self._name}.out.mannings.pfb')
+        return self._pfb_to_array(f"{self._name}.out.mannings.pfb")
 
     # ---------------------------------------------------------------------------
     # Mask
@@ -1493,7 +1579,7 @@ def mannings(self):
 
     @property
     def mask(self):
-        return self._pfb_to_array(f'{self._name}.out.mask.pfb')
+        return self._pfb_to_array(f"{self._name}.out.mask.pfb")
 
     # ---------------------------------------------------------------------------
     # Slopes X Y
@@ -1502,14 +1588,14 @@ def mask(self):
     @property
     def slope_x(self):
         if self._run.TopoSlopesX.FileName is None:
-            return self._pfb_to_array(f'{self._name}.out.slope_x.pfb')
+            return self._pfb_to_array(f"{self._name}.out.slope_x.pfb")
         else:
             return self._pfb_to_array(self._run.TopoSlopesX.FileName)
 
     @property
     def slope_y(self):
         if self._run.TopoSlopesY.FileName is None:
-            return self._pfb_to_array(f'{self._name}.out.slope_y.pfb')
+            return self._pfb_to_array(f"{self._name}.out.slope_y.pfb")
         else:
             return self._pfb_to_array(self._run.TopoSlopesY.FileName)
 
@@ -1520,7 +1606,7 @@ def slope_y(self):
     @property
     def elevation(self):
         if self._run.TopoSlopes.Elevation.FileName is None:
-            return self._pfb_to_array(f'{self._name}.DEM.pfb')
+            return self._pfb_to_array(f"{self._name}.DEM.pfb")
         else:
             return self._pfb_to_array(self._run.TopoSlopes.Elevation.FileName)
 
@@ -1530,7 +1616,7 @@ def elevation(self):
 
     @property
     def computed_porosity(self):
-        return self._pfb_to_array(f'{self._name}.out.porosity.pfb')
+        return self._pfb_to_array(f"{self._name}.out.porosity.pfb")
 
     # ---------------------------------------------------------------------------
     # Computed Permeability
@@ -1538,15 +1624,15 @@ def computed_porosity(self):
 
     @property
     def computed_permeability_x(self):
-        return self._pfb_to_array(f'{self._name}.out.perm_x.pfb')
+        return self._pfb_to_array(f"{self._name}.out.perm_x.pfb")
 
     @property
     def computed_permeability_y(self):
-        return self._pfb_to_array(f'{self._name}.out.perm_y.pfb')
+        return self._pfb_to_array(f"{self._name}.out.perm_y.pfb")
 
     @property
     def computed_permeability_z(self):
-        return self._pfb_to_array(f'{self._name}.out.perm_z.pfb')
+        return self._pfb_to_array(f"{self._name}.out.perm_z.pfb")
 
     # ---------------------------------------------------------------------------
     # Pressures
@@ -1555,16 +1641,16 @@ def computed_permeability_z(self):
     @property
     def pressure_initial_condition(self):
         press_type = self._run.ICPressure.Type
-        if press_type == 'PFBFile':
+        if press_type == "PFBFile":
             geom_name = self._run.ICPressure.GeomNames
             if len(geom_name) > 1:
-                msg = f'ICPressure.GeomNames are set to {geom_name}'
+                msg = f"ICPressure.GeomNames are set to {geom_name}"
                 raise Exception(msg)
             file_name = self._run.Geom[geom_name[0]].ICPressure.FileName
             return self._pfb_to_array(file_name)
         else:
             # HydroStaticPatch, ... ?
-            msg = f'Initial pressure of type {press_type} is not supported'
+            msg = f"Initial pressure of type {press_type} is not supported"
             raise Exception(msg)
 
     # ---------------------------------------------------------------------------
@@ -1585,7 +1671,7 @@ def pressure_boundary_conditions(self):
             cycle_name = self._run.Patch[p_name].BCPressure.Cycle
             cycle_names = self._run.Cycle[cycle_name].Names
             for c_name in cycle_names:
-                key = f'{p_name}__{c_name}'
+                key = f"{p_name}__{c_name}"
                 bc[key] = self._run.Patch[p_name].BCPressure[c_name].Value
 
         return bc
@@ -1594,7 +1680,7 @@ def pressure_boundary_conditions(self):
 
     @property
     def pressure(self):
-        file_name = get_absolute_path(f'{self._name}.out.press.{self._ts}.pfb')
+        file_name = get_absolute_path(f"{self._name}.out.press.{self._ts}.pfb")
         return self._pfb_to_array(file_name)
 
     # ---------------------------------------------------------------------------
@@ -1603,7 +1689,7 @@ def pressure(self):
 
     @property
     def saturation(self):
-        file_name = get_absolute_path(f'{self._name}.out.satur.{self._ts}.pfb')
+        file_name = get_absolute_path(f"{self._name}.out.satur.{self._ts}.pfb")
         return self._pfb_to_array(file_name)
 
     # ---------------------------------------------------------------------------
@@ -1612,7 +1698,7 @@ def saturation(self):
 
     @property
     def specific_storage(self):
-        return self._pfb_to_array(f'{self._name}.out.specific_storage.pfb')
+        return self._pfb_to_array(f"{self._name}.out.specific_storage.pfb")
 
     # ---------------------------------------------------------------------------
     # Evapotranspiration
@@ -1622,19 +1708,25 @@ def specific_storage(self):
     def et(self):
         if self._run.Solver.PrintCLM:
             # Read ET from CLM output
-            return self.clm_output('qflx_evap_tot')
+            return self.clm_output("qflx_evap_tot")
         else:
             # Assert that one and only one of Solver.EvapTransFile or Solver.EvapTransFileTransient is set
-            assert self._run.Solver.EvapTransFile != self._run.Solver.EvapTransFileTransient, \
-                'Only one of Solver.EvapTrans.FileName, Solver.EvapTransFileTransient can be set in order to ' \
-                'calculate evapotranspiration'
+            assert (
+                self._run.Solver.EvapTransFile
+                != self._run.Solver.EvapTransFileTransient
+            ), (
+                "Only one of Solver.EvapTrans.FileName, Solver.EvapTransFileTransient can be set in order to "
+                "calculate evapotranspiration"
+            )
 
             if self._run.Solver.EvapTransFile:
                 # Read steady-state flux file
                 et_data = self._pfb_to_array(self._run.Solver.EvapTrans.FileName)
             else:
                 # Read current timestep from series of flux PFB files
-                et_data = self._pfb_to_array(f'{self._run.Solver.EvapTrans.FileName}.{self._ts}.pfb')
+                et_data = self._pfb_to_array(
+                    f"{self._run.Solver.EvapTrans.FileName}.{self._ts}.pfb"
+                )
 
         return calculate_evapotranspiration(et_data, self.dx, self.dy, self.dz)
 
@@ -1642,17 +1734,35 @@ def et(self):
     # Overland Flow
     # ---------------------------------------------------------------------------
 
-    def overland_flow(self, flow_method='OverlandKinematic', epsilon=1e-5):
-        return calculate_overland_flow(self.pressure, self.slope_x, self.slope_y, self.mannings,
-                                       self.dx, self.dy, flow_method=flow_method, epsilon=epsilon, mask=self.mask)
+    def overland_flow(self, flow_method="OverlandKinematic", epsilon=1e-5):
+        return calculate_overland_flow(
+            self.pressure,
+            self.slope_x,
+            self.slope_y,
+            self.mannings,
+            self.dx,
+            self.dy,
+            flow_method=flow_method,
+            epsilon=epsilon,
+            mask=self.mask,
+        )
 
     # ---------------------------------------------------------------------------
     # Overland Flow Grid
     # ---------------------------------------------------------------------------
 
-    def overland_flow_grid(self, flow_method='OverlandKinematic', epsilon=1e-5):
-        return calculate_overland_flow_grid(self.pressure, self.slope_x, self.slope_y, self.mannings,
-                                            self.dx, self.dy, flow_method=flow_method, epsilon=epsilon, mask=self.mask)
+    def overland_flow_grid(self, flow_method="OverlandKinematic", epsilon=1e-5):
+        return calculate_overland_flow_grid(
+            self.pressure,
+            self.slope_x,
+            self.slope_y,
+            self.mannings,
+            self.dx,
+            self.dy,
+            flow_method=flow_method,
+            epsilon=epsilon,
+            mask=self.mask,
+        )
 
     # ---------------------------------------------------------------------------
     # Subsurface Storage
@@ -1660,8 +1770,16 @@ def overland_flow_grid(self, flow_method='OverlandKinematic', epsilon=1e-5):
 
     @property
     def subsurface_storage(self):
-        return calculate_subsurface_storage(self.computed_porosity, self.pressure, self.saturation,
-                                            self.specific_storage, self.dx, self.dy, self.dz, mask=self.mask)
+        return calculate_subsurface_storage(
+            self.computed_porosity,
+            self.pressure,
+            self.saturation,
+            self.specific_storage,
+            self.dx,
+            self.dy,
+            self.dz,
+            mask=self.mask,
+        )
 
     # ---------------------------------------------------------------------------
     # Surface Storage
@@ -1669,7 +1787,9 @@ def subsurface_storage(self):
 
     @property
     def surface_storage(self):
-        return calculate_surface_storage(self.pressure, self.dx, self.dy, mask=self.mask)
+        return calculate_surface_storage(
+            self.pressure, self.dx, self.dy, mask=self.mask
+        )
 
     # ---------------------------------------------------------------------------
     # Water Table Depth
@@ -1684,28 +1804,32 @@ def wtd(self):
     # ---------------------------------------------------------------------------
 
     def _clm_output_filepath(self, directory, prefix, ext):
-        file_name = f'{prefix}.{self._ts}.{ext}.{self._process_id}'
-        base_path = f'{self._run.Solver.CLM.CLMFileDir}/{directory}'
-        return get_absolute_path(f'{base_path}/{file_name}')
+        file_name = f"{prefix}.{self._ts}.{ext}.{self._process_id}"
+        base_path = f"{self._run.Solver.CLM.CLMFileDir}/{directory}"
+        return get_absolute_path(f"{base_path}/{file_name}")
 
     def _clm_output_bin(self, field, dtype):
-        fp = self._clm_output_filepath(field, field, 'bin')
-        return np.fromfile(fp, dtype=dtype, count=-1, sep='', offset=0)
+        fp = self._clm_output_filepath(field, field, "bin")
+        return np.fromfile(fp, dtype=dtype, count=-1, sep="", offset=0)
 
     def clm_output(self, field, layer=-1):
-        assert self._run.Solver.PrintCLM, 'CLM output must be enabled'
-        assert field in self.clm_output_variables, f'Unrecognized variable {field}'
+        assert self._run.Solver.PrintCLM, "CLM output must be enabled"
+        assert field in self.clm_output_variables, f"Unrecognized variable {field}"
 
         if self._run.Solver.CLM.SingleFile:
-            file_name = f'{self._name}.out.clm_output.{self._ts}.C.pfb'
-            arr = self._pfb_to_array(f'{file_name}')
+            file_name = f"{self._name}.out.clm_output.{self._ts}.C.pfb"
+            arr = self._pfb_to_array(f"{file_name}")
 
             nz = arr.shape[0]
-            nz_expected = len(self.clm_output_variables) + self._run.Solver.CLM.RootZoneNZ - 1
-            assert nz == nz_expected, f'Unexpected shape of CLM output, expected {nz_expected}, got {nz}'
+            nz_expected = (
+                len(self.clm_output_variables) + self._run.Solver.CLM.RootZoneNZ - 1
+            )
+            assert (
+                nz == nz_expected
+            ), f"Unexpected shape of CLM output, expected {nz_expected}, got {nz}"
 
             i = self.clm_output_variables.index(field)
-            if field == 't_soil':
+            if field == "t_soil":
                 if layer < 0:
                     i = layer
                 else:
@@ -1713,13 +1837,15 @@ def clm_output(self, field, layer=-1):
 
             arr = arr[i, :, :]
         else:
-            file_name = f'{self._name}.out.{field}.{self._ts}.pfb'
-            arr = self._pfb_to_array(f'{file_name}')
+            file_name = f"{self._name}.out.{field}.{self._ts}.pfb"
+            arr = self._pfb_to_array(f"{file_name}")
 
-            if field == 't_soil':
+            if field == "t_soil":
                 nz = arr.shape[0]
-                assert nz == self._run.Solver.CLM.RootZoneNZ, f'Unexpected shape of CLM output, expected ' \
-                                                              f'{self._run.Solver.CLM.RootZoneNZ}, got {nz}'
+                assert nz == self._run.Solver.CLM.RootZoneNZ, (
+                    f"Unexpected shape of CLM output, expected "
+                    f"{self._run.Solver.CLM.RootZoneNZ}, got {nz}"
+                )
                 arr = arr[layer, :, :]
 
         if arr.ndim == 3:
@@ -1728,137 +1854,138 @@ def clm_output(self, field, layer=-1):
 
     @property
     def clm_output_variables(self):
-        return ('eflx_lh_tot',
-                'eflx_lwrad_out',
-                'eflx_sh_tot',
-                'eflx_soil_grnd',
-                'qflx_evap_tot',
-                'qflx_evap_grnd',
-                'qflx_evap_soi',
-                'qflx_evap_veg',
-                'qflx_tran_veg',
-                'qflx_infl',
-                'swe_out',
-                't_grnd',
-                'qflx_qirr',
-                't_soil')
+        return (
+            "eflx_lh_tot",
+            "eflx_lwrad_out",
+            "eflx_sh_tot",
+            "eflx_soil_grnd",
+            "qflx_evap_tot",
+            "qflx_evap_grnd",
+            "qflx_evap_soi",
+            "qflx_evap_veg",
+            "qflx_tran_veg",
+            "qflx_infl",
+            "swe_out",
+            "t_grnd",
+            "qflx_qirr",
+            "t_soil",
+        )
 
     @property
     def clm_output_diagnostics(self):
-        return self._clm_output_filepath('diag_out', 'diagnostics', 'dat')
+        return self._clm_output_filepath("diag_out", "diagnostics", "dat")
 
     @property
     def clm_output_eflx_lh_tot(self):
-        return self._clm_output_bin('eflx_lh_tot', float)
+        return self._clm_output_bin("eflx_lh_tot", float)
 
     @property
     def clm_output_eflx_lwrad_out(self):
-        return self._clm_output_bin('eflx_lwrad_out', float)
+        return self._clm_output_bin("eflx_lwrad_out", float)
 
     @property
     def clm_output_eflx_sh_tot(self):
-        return self._clm_output_bin('eflx_sh_tot', float)
+        return self._clm_output_bin("eflx_sh_tot", float)
 
     @property
     def clm_output_eflx_soil_grnd(self):
-        return self._clm_output_bin('eflx_soil_grnd', float)
+        return self._clm_output_bin("eflx_soil_grnd", float)
 
     @property
     def clm_output_qflx_evap_grnd(self):
-        return self._clm_output_bin('qflx_evap_grnd', float)
+        return self._clm_output_bin("qflx_evap_grnd", float)
 
     @property
     def clm_output_qflx_evap_soi(self):
-        return self._clm_output_bin('qflx_evap_soi', float)
+        return self._clm_output_bin("qflx_evap_soi", float)
 
     @property
     def clm_output_qflx_evap_tot(self):
-        return self._clm_output_bin('qflx_evap_tot', float)
+        return self._clm_output_bin("qflx_evap_tot", float)
 
     @property
     def clm_output_qflx_evap_veg(self):
-        return self._clm_output_bin('qflx_evap_veg', float)
+        return self._clm_output_bin("qflx_evap_veg", float)
 
     @property
     def clm_output_qflx_infl(self):
-        return self._clm_output_bin('qflx_infl', float)
+        return self._clm_output_bin("qflx_infl", float)
 
     @property
     def clm_output_qflx_top_soil(self):
-        return self._clm_output_bin('qflx_top_soil', float)
+        return self._clm_output_bin("qflx_top_soil", float)
 
     @property
     def clm_output_qflx_tran_veg(self):
-        return self._clm_output_bin('qflx_tran_veg', float)
+        return self._clm_output_bin("qflx_tran_veg", float)
 
     @property
     def clm_output_swe_out(self):
-        return self._clm_output_bin('swe_out', float)
+        return self._clm_output_bin("swe_out", float)
 
     @property
     def clm_output_t_grnd(self):
-        return self._clm_output_bin('t_grnd', float)
+        return self._clm_output_bin("t_grnd", float)
 
     def clm_forcing(self, name):
         time_slice = self._run.Solver.CLM.MetFileNT
         prefix = self._run.Solver.CLM.MetFileName
         directory = self._run.Solver.CLM.MetFilePath
         file_index = int(self._forcing_time / time_slice)
-        t0 = f'{file_index * time_slice + 1:0>6}'
-        t1 = f'{(file_index + 1) * time_slice:0>6}'
-        file_name = get_absolute_path(
-            f'{directory}/{prefix}.{name}.{t0}_to_{t1}.pfb')
+        t0 = f"{file_index * time_slice + 1:0>6}"
+        t1 = f"{(file_index + 1) * time_slice:0>6}"
+        file_name = get_absolute_path(f"{directory}/{prefix}.{name}.{t0}_to_{t1}.pfb")
 
         return self._pfb_to_array(file_name)[self._forcing_time % time_slice]
 
     @property
     def clm_forcing_dswr(self):
         """Downward Visible or Short-Wave radiation [W/m2]"""
-        return self.clm_forcing('DSWR')
+        return self.clm_forcing("DSWR")
 
     @property
     def clm_forcing_dlwr(self):
         """Downward Infa-Red or Long-Wave radiation [W/m2]"""
-        return self.clm_forcing('DLWR')
+        return self.clm_forcing("DLWR")
 
     @property
     def clm_forcing_apcp(self):
         """Precipitation rate [mm/s]"""
-        return self.clm_forcing('APCP')
+        return self.clm_forcing("APCP")
 
     @property
     def clm_forcing_temp(self):
         """Air temperature [K]"""
-        return self.clm_forcing('Temp')
+        return self.clm_forcing("Temp")
 
     @property
     def clm_forcing_ugrd(self):
         """West-to-East or U-component of wind [m/s]"""
-        return self.clm_forcing('UGRD')
+        return self.clm_forcing("UGRD")
 
     @property
     def clm_forcing_vgrd(self):
         """South-to-North or V-component of wind [m/s]"""
-        return self.clm_forcing('VGRD')
+        return self.clm_forcing("VGRD")
 
     @property
     def clm_forcing_press(self):
         """Atmospheric Pressure [pa]"""
-        return self.clm_forcing('Press')
+        return self.clm_forcing("Press")
 
     @property
     def clm_forcing_spfh(self):
         """Water-vapor specific humidity [kg/kg]"""
-        return self.clm_forcing('SPFH')
+        return self.clm_forcing("SPFH")
 
     def _clm_map(self, root):
-        if root.Type == 'Constant':
+        if root.Type == "Constant":
             return root.Value
 
-        if root.Type == 'Linear':
+        if root.Type == "Linear":
             return (root.Min, root.Max)
 
-        if root.Type == 'PFBFile':
+        if root.Type == "PFBFile":
             return self._pfb_to_array(root.FileName)
 
         return None
diff --git a/pftools/python/parflow/tools/pf_backend.py b/pftools/python/parflow/tools/pf_backend.py
index 0ea429ff1..09a0cc2c0 100644
--- a/pftools/python/parflow/tools/pf_backend.py
+++ b/pftools/python/parflow/tools/pf_backend.py
@@ -13,7 +13,7 @@
 from .io import ParflowBinaryReader, read_pfb_sequence, read_pfb
 from collections.abc import Iterable
 from typing import Mapping, List, Union
-from xarray.backends  import BackendEntrypoint, BackendArray
+from xarray.backends import BackendEntrypoint, BackendArray
 from xarray.core import indexing
 from dask import delayed
 
@@ -35,7 +35,7 @@ class ParflowBackendEntrypoint(BackendEntrypoint):
         "read_inputs",
         "read_outputs",
         "inferred_dims",
-        "inferred_shape"
+        "inferred_shape",
     ]
 
     def open_dataset(
@@ -44,7 +44,7 @@ def open_dataset(
         *,
         base_dir=None,
         drop_variables=None,
-        name='parflow_variable',
+        name="parflow_variable",
         read_inputs=True,
         read_outputs=True,
         inferred_dims=None,
@@ -92,24 +92,23 @@ def open_dataset(
             An xr.Dataset with a collection of xr.DataArray objects as variables.
         """
         filetype = self.is_meta_or_pfb(filename_or_obj, strict=strict_ext_check)
-        if filetype == 'pfb':
+        if filetype == "pfb":
             # Reads a single pfb
             data = self.load_single_pfb(
-                      filename_or_obj,
-                      dims=inferred_dims,
-                      shape=inferred_shape)
+                filename_or_obj, dims=inferred_dims, shape=inferred_shape
+            )
             ds = xr.Dataset({name: data}).chunk(chunks)
-        elif filetype == 'pfmetadata':
+        elif filetype == "pfmetadata":
             # Reads full simulation input/output from pfmetadata
             if base_dir:
                 self.base_dir = base_dir
             else:
                 self.base_dir = os.path.dirname(filename_or_obj)
             ds = self.load_pfmetadata(
-                    filename_or_obj,
-                    self.pf_meta,
-                    read_inputs=read_inputs,
-                    read_outputs=read_outputs,
+                filename_or_obj,
+                self.pf_meta,
+                read_inputs=read_inputs,
+                read_outputs=read_outputs,
             )
         return ds
 
@@ -142,21 +141,21 @@ def load_pfmetadata(
             The assembled xarray dataset
         """
         ds = xr.Dataset()
-        ds.attrs['pf_metadata_file'] = filename_or_obj
-        ds.attrs['parflow_version'] = pf_meta['parflow']['build']['version']
-        if 'coordinates' in self.pf_meta:
-            coords = self.load_coords_from_meta(self.pf_meta['coordinates'])
+        ds.attrs["pf_metadata_file"] = filename_or_obj
+        ds.attrs["parflow_version"] = pf_meta["parflow"]["build"]["version"]
+        if "coordinates" in self.pf_meta:
+            coords = self.load_coords_from_meta(self.pf_meta["coordinates"])
             ds = ds.assign_coords(coords)
         if read_outputs:
-            for var, var_meta in self.pf_meta['outputs'].items():
+            for var, var_meta in self.pf_meta["outputs"].items():
                 if read_outputs is True or var in read_outputs:
                     das = self.load_pfb_from_meta(var_meta, name=var)
                     for k, v in das.items():
                         ds[k] = v
         if read_inputs:
-            for var, var_meta in self.pf_meta['inputs'].items():
-                if var == 'configuration':
-                    continue # TODO: Determine what to do with this
+            for var, var_meta in self.pf_meta["inputs"].items():
+                if var == "configuration":
+                    continue  # TODO: Determine what to do with this
                 if read_inputs is True or var in read_inputs:
                     das = self.load_pfb_from_meta(var_meta, name=var)
                     for k, v in das.items():
@@ -170,21 +169,23 @@ def load_coords_from_meta(self, coord_meta) -> Mapping[str, xr.DataArray]:
         """
         coords = {}
         for var, var_meta in coord_meta.items():
-            meta_type = var_meta['type']
-            if meta_type == 'time':
-                coords[var] = pd.DatetimeIndex(pd.date_range(
-                    start=var_meta['start'],
-                    end=var_meta['stop'],
-                    freq=var_meta['freq'],
-                ))
-            elif meta_type == 'pfb':
+            meta_type = var_meta["type"]
+            if meta_type == "time":
+                coords[var] = pd.DatetimeIndex(
+                    pd.date_range(
+                        start=var_meta["start"],
+                        end=var_meta["stop"],
+                        freq=var_meta["freq"],
+                    )
+                )
+            elif meta_type == "pfb":
                 coords[var] = self.load_pfb_from_meta(var_meta, name=var)[var]
             else:
-                #TODO: add a warning here
+                # TODO: add a warning here
                 pass
         return coords
 
-    def load_pfb_from_meta(self, var_meta, name='_') -> Mapping[str, xr.Dataset]:
+    def load_pfb_from_meta(self, var_meta, name="_") -> Mapping[str, xr.Dataset]:
         """
         Determines which sub-reader call to make based on the information
         in the metadata section  for a single variable of the pfmetadata file.
@@ -196,27 +197,27 @@ def load_pfb_from_meta(self, var_meta, name='_') -> Mapping[str, xr.Dataset]:
         :returns:
             A dictionary of xarray datasets with keys being the variable names
         """
-        base_type = var_meta['type']
-        if base_type == 'pfb':
+        base_type = var_meta["type"]
+        if base_type == "pfb":
             # Is it component?
-            if len(var_meta['data']) > 1 and 'component' in var_meta['data'][0]:
+            if len(var_meta["data"]) > 1 and "component" in var_meta["data"][0]:
                 ret_das = self.load_component_pfb(var_meta, name)
             # Is it time varying?
-            elif var_meta.get('time-varying', None):
+            elif var_meta.get("time-varying", None):
                 ret_das = self.load_time_varying_pfb(var_meta, name)
             # Is it normal
             else:
-                filename = var_meta['data'][0]['file']
+                filename = var_meta["data"][0]["file"]
                 if not os.path.exists(filename):
-                    filename = f'{self.base_dir}/{filename}'
+                    filename = f"{self.base_dir}/{filename}"
                 v = self.load_single_pfb(filename)
                 ret_das = {name: xr.Dataset({name: v})[name]}
-        elif base_type == 'clm_output':
+        elif base_type == "clm_output":
             ret_das = self.load_clm_output_pfb(var_meta, name)
-        elif base_type == 'pfb 2d timeseries':
+        elif base_type == "pfb 2d timeseries":
             ret_das = self.load_time_varying_2d_ts_pfb(var_meta, name)
         else:
-            raise ValueError(f'Could not find meta type for {base_type}')
+            raise ValueError(f"Could not find meta type for {base_type}")
         return ret_das
 
     def load_component_pfb(self, var_meta, name):
@@ -225,16 +226,16 @@ def load_component_pfb(self, var_meta, name):
         where component represents an anisotropy
         """
         all_da = {}
-        if var_meta['domain'] == 'surface':
-            dims = ('x', 'y')
-        elif var_meta['domain'] == 'subsurface':
-            dims = ('z', 'y', 'x')
-        for sub_dict in var_meta['data']:
-            component = sub_dict['component']
-            comp_name = f'{name}_{component}'
-            file = sub_dict['file']
+        if var_meta["domain"] == "surface":
+            dims = ("x", "y")
+        elif var_meta["domain"] == "subsurface":
+            dims = ("z", "y", "x")
+        for sub_dict in var_meta["data"]:
+            component = sub_dict["component"]
+            comp_name = f"{name}_{component}"
+            file = sub_dict["file"]
             if not os.path.exists(file):
-                file = f'{self.base_dir}/{file}'
+                file = f"{self.base_dir}/{file}"
             v = self.load_single_pfb(file)
             all_da[comp_name] = xr.Dataset({comp_name: v})[comp_name]
         return all_da
@@ -244,18 +245,18 @@ def load_time_varying_pfb(self, var_meta, name):
         These filetypes have dimensions (time, z, y, x)
         where a each file represents an individual time
         """
-        file_template = var_meta['data'][0]['file-series']
+        file_template = var_meta["data"][0]["file-series"]
         n_time = 0
-        concat_dim = 'time'
-        time_idx = np.arange(*var_meta['data'][0]['time-range'])
+        concat_dim = "time"
+        time_idx = np.arange(*var_meta["data"][0]["time-range"])
         n_time = time_idx[-1]
-        pad, fmt = file_template.split('.')[-2:]
-        basename = '.'.join(file_template.split('.')[:-2])
-        all_files = [f'{basename}.{pad%n}.{fmt}' for n in time_idx]
+        pad, fmt = file_template.split(".")[-2:]
+        basename = ".".join(file_template.split(".")[:-2])
+        all_files = [f"{basename}.{pad%n}.{fmt}" for n in time_idx]
         # Check if basename contains any of the files if not,
         # fall back to `self.base_dir` from the pfmetadata file
         if not os.path.exists(all_files[0]):
-            all_files = [f'{self.base_dir}/{af}' for af in all_files]
+            all_files = [f"{self.base_dir}/{af}" for af in all_files]
 
         # Put it all together
         base_da = self.load_sequence_of_pfb(all_files)
@@ -268,22 +269,23 @@ def load_time_varying_2d_ts_pfb(self, var_meta, name):
         where the time dimension will be strided along separate files
         and each individual file contains time_slice number of timesteps
         """
-        concat_dim = 'z' # z is time here
-        time_start = np.arange(*var_meta['data'][0]['times-between'])
-        time_end = time_start + var_meta['data'][0]['times-between'][-1] - 1
+        concat_dim = "z"  # z is time here
+        time_start = np.arange(*var_meta["data"][0]["times-between"])
+        time_end = time_start + var_meta["data"][0]["times-between"][-1] - 1
         ntime = time_end[-1]
-        file_template = var_meta['data'][0]['file-series']
-        pad, fmt = file_template.split('.')[-2:]
-        basename = '.'.join(file_template.split('.')[:-2])
-        all_files = [f'{basename}.{pad%(s,e)}.{fmt}'
-                     for s, e in zip(time_start, time_end)]
+        file_template = var_meta["data"][0]["file-series"]
+        pad, fmt = file_template.split(".")[-2:]
+        basename = ".".join(file_template.split(".")[:-2])
+        all_files = [
+            f"{basename}.{pad%(s,e)}.{fmt}" for s, e in zip(time_start, time_end)
+        ]
         # Check if basename contains any of the files if not,
         # fall back to `self.base_dir` from the pfmetadata file
         if not os.path.exists(all_files[0]):
-            all_files = [f'{self.base_dir}/{af}' for af in all_files]
+            all_files = [f"{self.base_dir}/{af}" for af in all_files]
 
         # Put it all together
-        base_da = self.load_sequence_of_pfb(all_files, z_is='time')
+        base_da = self.load_sequence_of_pfb(all_files, z_is="time")
         base_da = xr.Dataset({name: base_da})[name]
         return {name: base_da}
 
@@ -293,42 +295,43 @@ def load_clm_output_pfb(self, var_meta, name):
         where the variable ordering is fixed and each file represents an
         individual timestep
         """
-        warnings.warn("""
+        warnings.warn(
+            """
             Reading CLM output is not officially supported,
             at this time. We'll try our best to load the data,
             but this may break in the future!
             """
         )
         varnames = [
-            'latent_heat_flux',
-            'outgoing_longwave',
-            'sensible_heat_flux',
-            'ground_heat_flux',
-            'total_evapotranspiration',
-            'ground_evaporation',
-            'soil_evaporation',
-            'veg_evaporation',
-            'transpiration',
-            'infiltration',
-            'swe',
-            't_ground',
+            "latent_heat_flux",
+            "outgoing_longwave",
+            "sensible_heat_flux",
+            "ground_heat_flux",
+            "total_evapotranspiration",
+            "ground_evaporation",
+            "soil_evaporation",
+            "veg_evaporation",
+            "transpiration",
+            "infiltration",
+            "swe",
+            "t_ground",
         ]
-        file_template = var_meta['data'][0]['file-series']
+        file_template = var_meta["data"][0]["file-series"]
         n_time = 0
-        concat_dim = 'time'
-        time_idx = np.arange(*var_meta['data'][0]['time-range'])
+        concat_dim = "time"
+        time_idx = np.arange(*var_meta["data"][0]["time-range"])
         n_time = time_idx[-1]
-        pad, filler, fmt = file_template.split('.')[-3:]
-        basename = '.'.join(file_template.split('.')[:-3])
-        all_files = [f'{basename}.{pad%n}.{filler}.{fmt}' for n in time_idx]
+        pad, filler, fmt = file_template.split(".")[-3:]
+        basename = ".".join(file_template.split(".")[:-3])
+        all_files = [f"{basename}.{pad%n}.{filler}.{fmt}" for n in time_idx]
         # Check if basename contains any of the files if not,
         # fall back to `self.base_dir` from the pfmetadata file
         if not os.path.exists(all_files[0]):
-            all_files = [f'{self.base_dir}/{af}' for af in all_files]
+            all_files = [f"{self.base_dir}/{af}" for af in all_files]
 
         clm_das = []
         for i, v in enumerate(varnames):
-            var_da = self.load_sequence_of_pfb(all_files, init_key={'z': i})
+            var_da = self.load_sequence_of_pfb(all_files, init_key={"z": i})
             var_da = xr.Dataset({v: var_da})[v]
             clm_das.append(var_da)
         clm_das = xr.merge(clm_das)
@@ -340,34 +343,28 @@ def load_single_pfb(
         dims=None,
         shape=None,
         z_first=True,
-        z_is='z',
+        z_is="z",
     ) -> xr.Variable:
         """
         Load a `pfb` file directly as an xr.Variable
         """
         data = indexing.LazilyIndexedArray(
             ParflowBackendArray(
-                filename_or_obj,
-                dims=dims,
-                shape=shape,
-                z_first=z_first,
-                z_is=z_is
-        ))
+                filename_or_obj, dims=dims, shape=shape, z_first=z_first, z_is=z_is
+            )
+        )
         if not dims:
             dims = data.array.dims
         if not shape:
             shape = data.array.shape
-        var = xr.Variable(dims, data, )
+        var = xr.Variable(
+            dims,
+            data,
+        )
         return var
 
     def load_sequence_of_pfb(
-        self,
-        filenames,
-        dims=None,
-        shape=None,
-        z_first=True,
-        z_is='z',
-        init_key={}
+        self, filenames, dims=None, shape=None, z_first=True, z_is="z", init_key={}
     ) -> xr.Variable:
         data = indexing.LazilyIndexedArray(
             ParflowBackendArray(
@@ -377,7 +374,8 @@ def load_sequence_of_pfb(
                 z_first=z_first,
                 z_is=z_is,
                 init_key=init_key,
-        ))
+            )
+        )
         if not dims:
             dims = data.array.dims
         if not shape:
@@ -387,15 +385,18 @@ def load_sequence_of_pfb(
 
     def is_meta_or_pfb(self, filename_or_obj, strict=True):
         """Determine if a file is a pfb file or pfmetadata file"""
+
         def _check_dict_is_valid_meta(meta):
-            assert 'parflow' in meta.keys(), \
-                ('Metadata file missing "parflow" key - ',
-                 'are you sure this is a valid Parflow metadata file?')
+            assert "parflow" in meta.keys(), (
+                'Metadata file missing "parflow" key - ',
+                "are you sure this is a valid Parflow metadata file?",
+            )
+
         if not strict:
             # Just check the extension
-            ext = filename_or_obj.split('.')[-1]
-            if ext == 'pfmetadata':
-                with open(filename_or_obj, 'r') as f:
+            ext = filename_or_obj.split(".")[-1]
+            if ext == "pfmetadata":
+                with open(filename_or_obj, "r") as f:
                     pf_meta = json.load(f)
                     _check_dict_is_valid_meta(pf_meta)
                     self.pf_meta = pf_meta
@@ -404,44 +405,41 @@ def _check_dict_is_valid_meta(meta):
         if isinstance(filename_or_obj, str):
             try:
                 with ParflowBinaryReader(filename_or_obj) as pfd:
-                    assert 'nx' in pfd.header
-                    assert 'ny' in pfd.header
-                    assert 'nz' in pfd.header
-                return 'pfb'
+                    assert "nx" in pfd.header
+                    assert "ny" in pfd.header
+                    assert "nz" in pfd.header
+                return "pfb"
             except AssertionError:
-                with open(filename_or_obj, 'r') as f:
+                with open(filename_or_obj, "r") as f:
                     pf_meta = json.load(f)
                     _check_dict_is_valid_meta(pf_meta)
                     self.pf_meta = pf_meta
-                    return 'pfmetadata'
+                    return "pfmetadata"
         elif isinstance(filename_or_obj, dict):
             _check_dict_is_valid_meta(filename_or_obj)
             self.pf_meta = filename_or_obj
-            return 'pfmetadata'
+            return "pfmetadata"
         else:
             raise NotImplementedError("Was unable to determine input type!")
 
-    def _infer_dims_and_shape(self, file, z_first=True, z_is='z'):
+    def _infer_dims_and_shape(self, file, z_first=True, z_is="z"):
         """Determine the dimensions and shape of a pfb file"""
-        pfd = xr.Dataset(
-            {'_': self.load_single_pfb(file, z_first=z_first, z_is=z_is)}
-        )
+        pfd = xr.Dataset({"_": self.load_single_pfb(file, z_first=z_first, z_is=z_is)})
         dims = list(pfd.dims.keys())
         shape = list(pfd.dims.values())
         del pfd
         return dims, shape
 
-
     def guess_can_open(self, filename_or_obj):
         """Registers the backend to recognize *.pfb and *.pfmetadata files"""
-        openable_extensions = ['pfb', 'pfmetadata']
+        openable_extensions = ["pfb", "pfmetadata"]
         for ext in openable_extensions:
             if filename_or_obj.endswith(ext):
                 return True
         return False
 
 
-def _getitem_no_state(file_or_seq, key, dims, mode, z_first=True, z_is='z'):
+def _getitem_no_state(file_or_seq, key, dims, mode, z_first=True, z_is="z"):
     """
     Base functionality for actually getting data out of PFB files.
 
@@ -461,29 +459,31 @@ def _getitem_no_state(file_or_seq, key, dims, mode, z_first=True, z_is='z'):
     :return:
         A numpy array of the data
     """
-    if mode == 'single':
-        accessor = {d: util._key_to_explicit_accessor(k)
-                    for d, k in zip(dims, key)}
+    if mode == "single":
+        accessor = {d: util._key_to_explicit_accessor(k) for d, k in zip(dims, key)}
         sub = read_pfb(
             file_or_seq,
             keys=accessor,
             z_first=z_first,
         )
-    elif mode == 'sequence':
-        accessor = {d: util._key_to_explicit_accessor(k)
-                for d, k in zip(dims, key)}
-        t_start = accessor['time']['start']
-        t_end = accessor['time']['stop']
-        if z_is == 'time':
+    elif mode == "sequence":
+        accessor = {d: util._key_to_explicit_accessor(k) for d, k in zip(dims, key)}
+        t_start = accessor["time"]["start"]
+        t_end = accessor["time"]["stop"]
+        if z_is == "time":
             # WARNING:  This is pretty hacky, accounting for first timestep offset
             try:
                 # Parflow files end with TIMESTEP.pfb
-                file_start_time = int(file_or_seq[t_start].split('.')[-2].split('_')[0]) - 1
+                file_start_time = (
+                    int(file_or_seq[t_start].split(".")[-2].split("_")[0]) - 1
+                )
             except:
                 # CLM output files end with TIMESTEP.C.pfb
-                file_start_time = int(file_or_seq[t_start].split('.')[-3].split('_')[0]) - 1
-            accessor['time']['start'] -= file_start_time
-            accessor['time']['stop'] -= file_start_time
+                file_start_time = (
+                    int(file_or_seq[t_start].split(".")[-3].split("_")[0]) - 1
+                )
+            accessor["time"]["start"] -= file_start_time
+            accessor["time"]["stop"] -= file_start_time
         if t_start is not None and t_start == t_end:
             t_end += 1
 
@@ -492,8 +492,8 @@ def _getitem_no_state(file_or_seq, key, dims, mode, z_first=True, z_is='z'):
         # out of the list we must remove the accessor indices
         # because they have been used.
         read_files = file_or_seq[t_start:t_end]
-        read_files = np.array(read_files)[accessor['time']['indices']]
-        accessor['time']['indices'] = slice(None, None, None)
+        read_files = np.array(read_files)[accessor["time"]["indices"]]
+        accessor["time"]["indices"] = slice(None, None, None)
         # Read the array
         sub = read_pfb_sequence(
             read_files,
@@ -501,13 +501,12 @@ def _getitem_no_state(file_or_seq, key, dims, mode, z_first=True, z_is='z'):
             z_first=z_first,
             z_is=z_is,
         )
-    sub = sub[tuple([accessor[d]['indices'] for d in dims])]
+    sub = sub[tuple([accessor[d]["indices"] for d in dims])]
     # Check which axes need to be squeezed out. This is
     # to distinguish between doing `ds.isel(x=[0])` which
     # should keep the x axis (and dimension) and `ds.isel(x=0)`
     # which should remove the x axis (and dimension).
-    axes_to_squeeze = tuple(i for i, d in enumerate(dims)
-                            if accessor[d]['squeeze'])
+    axes_to_squeeze = tuple(i for i, d in enumerate(dims) if accessor[d]["squeeze"])
     sub = np.squeeze(sub, axis=axes_to_squeeze)
     return sub
 
@@ -516,13 +515,13 @@ class ParflowBackendArray(BackendArray):
     """Backend array that allows for lazy indexing on pfb-based data."""
 
     def __init__(
-         self,
-         file_or_seq,
-         dims=None,
-         shape=None,
-         z_first=True,
-         z_is='z',
-         init_key={},
+        self,
+        file_or_seq,
+        dims=None,
+        shape=None,
+        z_first=True,
+        z_is="z",
+        init_key={},
     ):
         """
         Instantiate a new ParflowBackendArray.
@@ -542,14 +541,14 @@ def __init__(
         """
         self.file_or_seq = file_or_seq
         if isinstance(self.file_or_seq, str):
-            self.mode = 'single'
+            self.mode = "single"
             self.header_file = self.file_or_seq
         elif isinstance(self.file_or_seq, Iterable):
-            self.mode = 'sequence'
+            self.mode = "sequence"
             self.header_file = self.file_or_seq[0]
             # TODO: Should this be done in `load_time_varying_2d_ts_pfb`?
-            if z_is == 'time' and shape is not None:
-                time_idx = np.nonzero(np.array(dims) == 'time')[0][0]
+            if z_is == "time" and shape is not None:
+                time_idx = np.nonzero(np.array(dims) == "time")[0][0]
                 ntime = np.array(shape)[time_idx]
                 ts_per_file = int(ntime / len(self.file_or_seq))
                 self.file_or_seq = np.repeat(self.file_or_seq, ts_per_file)
@@ -565,9 +564,7 @@ def __init__(
         # to have valid `nbytes` attribute
         self.dtype = np.dtype(np.float64)
 
-    def __getitem__(
-            self, key: xr.core.indexing.ExplicitIndexer
-    ) -> np.ndarray:
+    def __getitem__(self, key: xr.core.indexing.ExplicitIndexer) -> np.ndarray:
         """Dunder method to call implement the underlying indexing scheme"""
         return indexing.explicit_indexing_adapter(
             key,
@@ -578,26 +575,25 @@ def __getitem__(
 
     def _set_dims_and_shape(self):
         with ParflowBinaryReader(
-            self.header_file,
-            precompute_subgrid_info=False
+            self.header_file, precompute_subgrid_info=False
         ) as pfd:
             if self.z_first:
-                _shape = [pfd.header['nz'], pfd.header['ny'], pfd.header['nx']]
+                _shape = [pfd.header["nz"], pfd.header["ny"], pfd.header["nx"]]
             else:
-                _shape = [pfd.header['nx'], pfd.header['ny'], pfd.header['nz']]
-        if self.mode == 'sequence':
+                _shape = [pfd.header["nx"], pfd.header["ny"], pfd.header["nz"]]
+        if self.mode == "sequence":
             _shape = [len(self.file_or_seq), *_shape]
         # Construct dimension template
-        if self.mode == 'single':
+        if self.mode == "single":
             if self.z_first:
-                _dims = ['z', 'y', 'x']
+                _dims = ["z", "y", "x"]
             else:
-                _dims = ['x', 'y', 'z']
-        elif self.mode == 'sequence':
+                _dims = ["x", "y", "z"]
+        elif self.mode == "sequence":
             if self.z_first:
-                _dims = ['time', 'z', 'y', 'x']
+                _dims = ["time", "z", "y", "x"]
             else:
-                _dims = ['time', 'x', 'y', 'x']
+                _dims = ["time", "x", "y", "x"]
         # Add some logic for dealing with clm output's inconsistent format
         if self.init_key:
             for i, (dim, size) in enumerate(zip(_dims, _shape)):
@@ -650,8 +646,8 @@ def _getitem(self, key: tuple) -> np.ndarray:
         """Mapping between keys to the actual data"""
         real_size = self._size_from_key(key)
         sub = delayed(_getitem_no_state)(
-                self.file_or_seq, key, self.dims, self.mode,
-                self.z_first, self.z_is)
+            self.file_or_seq, key, self.dims, self.mode, self.z_first, self.z_is
+        )
         sub = dask.array.from_delayed(sub, self.pfb_shape, dtype=np.float64)
         if self.shape != sub.shape:
             sub = dask.array.squeeze(sub, axis=self.squeeze_dims)
diff --git a/pftools/python/parflow/tools/ref/reservoir_keys.yaml b/pftools/python/parflow/tools/ref/reservoir_keys.yaml
new file mode 100644
index 000000000..1ba0b25b9
--- /dev/null
+++ b/pftools/python/parflow/tools/ref/reservoir_keys.yaml
@@ -0,0 +1,101 @@
+# -----------------------------------------------------------------------------
+# Definition anatomy
+# -----------------------------------------------------------------------------
+#
+# run.Geom.{name}.{PARFLOW_KEY}:
+#   alias:
+#     - Short names that can be used inside your table definition
+#   addon:
+#     If provided those other key/value will be set under
+#       Geom.{name}.{KEY_1} = {VALUE_1}
+#       Geom.{name}.{KEY_2} = {VALUE_2}
+#   register:
+#     If provided the {name} will be added to a set() with the name
+#     run.{VALUE} = [] + {name}.
+#
+# -----------------------------------------------------------------------------
+
+key:
+  type: str
+  alias:
+    - key
+    - name
+    - Name
+    - Key
+
+# -----------------------------------------------------------------------------
+# Reservoir Parameters
+# -----------------------------------------------------------------------------
+
+Release_X:
+  type: float
+  alias:
+    - Release_X
+  register: Reservoirs.Names
+
+Release_Y:
+  type: float
+  alias:
+    - Release_Y
+  register: Reservoirs.Names
+
+Intake_X:
+  type: float
+  alias:
+    - Intake_X
+  register: Reservoirs.Names
+
+Intake_Y:
+  type: float
+  alias:
+    - Intake_Y
+  register: Reservoirs.Names
+
+Secondary_Intake_X:
+  type: float
+  alias:
+    - Secondary_Intake_X
+  register: Reservoirs.Names
+
+Secondary_Intake_Y:
+  type: float
+  alias:
+    - Secondary_Intake_Y
+  register: Reservoirs.Names
+
+Has_Secondary_Intake_Cell:
+  type: int
+  alias:
+    - Has_Secondary_Intake_Cell
+  register: Reservoirs.Names
+
+Min_Release_Storage:
+  type: float
+  alias:
+    - Min_Release_Storage
+  register: Reservoirs.Names
+
+Release_Rate:
+  type: float
+  alias:
+    - Release_Rate
+  register: Reservoirs.Names
+
+Max_Storage:
+  type: float
+  alias:
+    - Max_Storage
+  register: Reservoirs.Names
+
+Storage:
+  type: float
+  alias:
+    - Storage
+  register: Reservoirs.Names
+
+Overland_Flow_Solver:
+  # typos - vertical
+  type: string
+  alias:
+    - Overland_Flow_Solver
+  register: Reservoirs.Names
diff --git a/pftools/python/parflow/tools/settings.py b/pftools/python/parflow/tools/settings.py
index c7932fc74..03c8154a0 100644
--- a/pftools/python/parflow/tools/settings.py
+++ b/pftools/python/parflow/tools/settings.py
@@ -7,17 +7,19 @@
 WORKING_DIRECTORY = os.getcwd()
 PRINT_LINE_ERROR = False
 EXIT_ON_ERROR = False
-PARFLOW_VERSION = '3.6.0'
+PARFLOW_VERSION = "3.6.0"
 
 
 # ---------------------------------------------------------------------------
 
+
 def get_working_directory():
     return WORKING_DIRECTORY
 
 
 # ---------------------------------------------------------------------------
 
+
 def set_working_directory(new_working_directory=None):
     """This will set the working directory to use for all the
     relative file path.
@@ -32,6 +34,7 @@ def set_working_directory(new_working_directory=None):
 
 # ---------------------------------------------------------------------------
 
+
 def enable_line_error():
     """Calling that method will enable line feedback on validation
     error
@@ -65,7 +68,6 @@ def disable_exit_error():
 
 
 def set_parflow_version(version):
-    """Globally store the ParFlow version to test against
-    """
+    """Globally store the ParFlow version to test against"""
     global PARFLOW_VERSION
     PARFLOW_VERSION = version
diff --git a/pftools/python/parflow/tools/terminal.py b/pftools/python/parflow/tools/terminal.py
index b6980d583..d3206ba2a 100644
--- a/pftools/python/parflow/tools/terminal.py
+++ b/pftools/python/parflow/tools/terminal.py
@@ -7,29 +7,29 @@
 
 
 class Colors:
-    """Terminal color helper
-    """
-    HEADER = '\033[95m'
-    OKBLUE = '\033[94m'
-    OKGREEN = '\033[92m'
-    WARNING = '\033[93m'
-    FAIL = '\033[91m'
-    ENDC = '\033[0m'
-    BOLD = '\033[1m'
-    UNDERLINE = '\033[4m'
-    BLUE = '\033[34m'
-    MAGENTA = '\033[35m'
-    CYAN = '\033[36m'
+    """Terminal color helper"""
+
+    HEADER = "\033[95m"
+    OKBLUE = "\033[94m"
+    OKGREEN = "\033[92m"
+    WARNING = "\033[93m"
+    FAIL = "\033[91m"
+    ENDC = "\033[0m"
+    BOLD = "\033[1m"
+    UNDERLINE = "\033[4m"
+    BLUE = "\033[34m"
+    MAGENTA = "\033[35m"
+    CYAN = "\033[36m"
 
 
 class Symbols:
-    """Terminal emoji helper
-    """
-    ok = u'\u2714'
-    ko = u'\u2718'
-    errorItem = u'\u2605'
-    warning = u'\u26A0'
-    default = u'\u2622'
-    droplet = u'\U0001F4A7'
-    splash = u'\U0001F4A6'
-    x = u'\u274C'
+    """Terminal emoji helper"""
+
+    ok = "\u2714"
+    ko = "\u2718"
+    errorItem = "\u2605"
+    warning = "\u26A0"
+    default = "\u2622"
+    droplet = "\U0001F4A7"
+    splash = "\U0001F4A6"
+    x = "\u274C"
diff --git a/pftools/python/parflow/tools/tests/pfb_summary.py b/pftools/python/parflow/tools/tests/pfb_summary.py
index 6f101e22e..faa8367fe 100644
--- a/pftools/python/parflow/tools/tests/pfb_summary.py
+++ b/pftools/python/parflow/tools/tests/pfb_summary.py
@@ -11,6 +11,7 @@
 
         The purpose of this utility is to assist with debugging so you can view a summary of a PFB file.
 """
+
 from ctypes import cdll
 import sys
 import os
@@ -68,7 +69,7 @@ def print_summary(self):
         checksum = 0
         first_cell = 0
         printed_dots = False
-        number_of_subgrids = int(self.header.get('n_subgrids', 0))
+        number_of_subgrids = int(self.header.get("n_subgrids", 0))
         for i in range(0, number_of_subgrids):
             subgrid_header = self.read_subgrid_header()
             data = self.read_subgrid_data(subgrid_header)
@@ -104,56 +105,56 @@ def read_header(self):
 
         self.fp.seek(0)
         self.header = {}
-        self.header['x'] = struct.unpack('>d', self.fp.read(8))[0]
-        self.header['y'] = struct.unpack('>d', self.fp.read(8))[0]
-        self.header['z'] = struct.unpack('>d', self.fp.read(8))[0]
-        self.header['nx'] = struct.unpack('>i', self.fp.read(4))[0]
-        self.header['ny'] = struct.unpack('>i', self.fp.read(4))[0]
-        self.header['nz'] = struct.unpack('>i', self.fp.read(4))[0]
-        self.header['dx'] = struct.unpack('>d', self.fp.read(8))[0]
-        self.header['dy'] = struct.unpack('>d', self.fp.read(8))[0]
-        self.header['dz'] = struct.unpack('>d', self.fp.read(8))[0]
-        self.header['n_subgrids'] = struct.unpack('>i', self.fp.read(4))[0]
+        self.header["x"] = struct.unpack(">d", self.fp.read(8))[0]
+        self.header["y"] = struct.unpack(">d", self.fp.read(8))[0]
+        self.header["z"] = struct.unpack(">d", self.fp.read(8))[0]
+        self.header["nx"] = struct.unpack(">i", self.fp.read(4))[0]
+        self.header["ny"] = struct.unpack(">i", self.fp.read(4))[0]
+        self.header["nz"] = struct.unpack(">i", self.fp.read(4))[0]
+        self.header["dx"] = struct.unpack(">d", self.fp.read(8))[0]
+        self.header["dy"] = struct.unpack(">d", self.fp.read(8))[0]
+        self.header["dz"] = struct.unpack(">d", self.fp.read(8))[0]
+        self.header["n_subgrids"] = struct.unpack(">i", self.fp.read(4))[0]
 
     def read_subgrid_header(self):
         """Read the subgrid header from the file and return the header as a dict."""
 
         subgrid_header = {}
-        subgrid_header['ix'] = struct.unpack('>i', self.fp.read(4))[0]
-        subgrid_header['iy'] = struct.unpack('>i', self.fp.read(4))[0]
-        subgrid_header['iz'] = struct.unpack('>i', self.fp.read(4))[0]
-        subgrid_header['nx'] = struct.unpack('>i', self.fp.read(4))[0]
-        subgrid_header['ny'] = struct.unpack('>i', self.fp.read(4))[0]
-        subgrid_header['nz'] = struct.unpack('>i', self.fp.read(4))[0]
-        subgrid_header['rx'] = struct.unpack('>i', self.fp.read(4))[0]
-        subgrid_header['ry'] = struct.unpack('>i', self.fp.read(4))[0]
-        subgrid_header['rz'] = struct.unpack('>i', self.fp.read(4))[0]
+        subgrid_header["ix"] = struct.unpack(">i", self.fp.read(4))[0]
+        subgrid_header["iy"] = struct.unpack(">i", self.fp.read(4))[0]
+        subgrid_header["iz"] = struct.unpack(">i", self.fp.read(4))[0]
+        subgrid_header["nx"] = struct.unpack(">i", self.fp.read(4))[0]
+        subgrid_header["ny"] = struct.unpack(">i", self.fp.read(4))[0]
+        subgrid_header["nz"] = struct.unpack(">i", self.fp.read(4))[0]
+        subgrid_header["rx"] = struct.unpack(">i", self.fp.read(4))[0]
+        subgrid_header["ry"] = struct.unpack(">i", self.fp.read(4))[0]
+        subgrid_header["rz"] = struct.unpack(">i", self.fp.read(4))[0]
         return subgrid_header
 
     def read_subgrid_data(self, subgrid_header):
         """Read the data of the subgrid. Returns a numpy array mapped to the subgrid data."""
-        ix = subgrid_header['ix']
-        iy = subgrid_header['iy']
-        iz = subgrid_header['iz']
-        nx = subgrid_header['nx']
-        ny = subgrid_header['ny']
-        nz = subgrid_header['nz']
+        ix = subgrid_header["ix"]
+        iy = subgrid_header["iy"]
+        iz = subgrid_header["iz"]
+        nx = subgrid_header["nx"]
+        ny = subgrid_header["ny"]
+        nz = subgrid_header["nz"]
 
         offset = self.fp.tell()
         shape = [nz, ny, nx]
         data = np.memmap(
             self.file_name,
             dtype=np.float64,
-            mode='r',
+            mode="r",
             offset=offset,
             shape=tuple(shape),
-            order='F'
+            order="F",
         ).byteswap()
-        offset = offset + nx*ny*nz*8
+        offset = offset + nx * ny * nz * 8
         self.fp.seek(offset)
         return data
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main = PFBSummary()
     main.run()
diff --git a/pftools/python/parflow/tools/tests/test_pf_xarray.py b/pftools/python/parflow/tools/tests/test_pf_xarray.py
index 984158f21..45425194a 100644
--- a/pftools/python/parflow/tools/tests/test_pf_xarray.py
+++ b/pftools/python/parflow/tools/tests/test_pf_xarray.py
@@ -4,13 +4,13 @@
     It uses existing test data .pfb files in the parflow repository for testing.
 """
 
-
 import sys
 import os
 import unittest
 import numpy as np
 import xarray as xr
 import tempfile
+
 rootdir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
 sys.path.append(rootdir)
 from parflow.tools.pf_backend import ParflowBackendEntrypoint
@@ -27,9 +27,7 @@ def test_open_dataset(self):
         """Test reading a pfb file using xarray open_dataset."""
 
         ds = xr.open_dataset(
-            EXAMPLE_PFB_FILE_PATH_0,
-            name='forsyth5.out.press',
-            engine='parflow'
+            EXAMPLE_PFB_FILE_PATH_0, name="forsyth5.out.press", engine="parflow"
         )
 
         # Verify the sizes of the dimensions from the loaded xarray
@@ -46,10 +44,7 @@ def test_open_dataset(self):
     def test_read_pfb_file_list(self):
         """Test reading a list of pfb files."""
 
-        pfb_file_list = [
-            EXAMPLE_PFB_FILE_PATH_0,
-            EXAMPLE_PFB_FILE_PATH_1
-        ]
+        pfb_file_list = [EXAMPLE_PFB_FILE_PATH_0, EXAMPLE_PFB_FILE_PATH_1]
         da = read_pfb_sequence(pfb_file_list)
 
         # Verify that the shape of the data set is 2 time rows with the expected 3D dimensions
@@ -64,33 +59,33 @@ def test_write_pfb_file(self):
         with ParflowBinaryReader(EXAMPLE_PFB_FILE_PATH_0) as pfb:
             da = pfb.read_all_subgrids()
             header = pfb.header
-            header['p'] = 8
-            header['q'] = 5
-            header['r'] = 1
-            header['n_subgrids'] = header['p'] * header['q'] * header['r']
+            header["p"] = 8
+            header["q"] = 5
+            header["r"] = 1
+            header["n_subgrids"] = header["p"] * header["q"] * header["r"]
 
             with tempfile.TemporaryDirectory() as TEMP_DIRECTORY:
-                file_name = f'{TEMP_DIRECTORY}/example0.pfb'
+                file_name = f"{TEMP_DIRECTORY}/example0.pfb"
                 write_pfb(file_name, da, **header)
                 total = 0
                 with PFBSummary() as s:
                     header = s.open_pfb(file_name)
-                    self.assertEqual(46, header.get('nx'))
-                    self.assertEqual(46, header.get('ny'))
-                    self.assertEqual(21, header.get('nz'))
-                    number_of_subgrids = int(header.get('n_subgrids', 0))
+                    self.assertEqual(46, header.get("nx"))
+                    self.assertEqual(46, header.get("ny"))
+                    self.assertEqual(21, header.get("nz"))
+                    number_of_subgrids = int(header.get("n_subgrids", 0))
                     self.assertEqual(40, number_of_subgrids)
                     for i in range(0, number_of_subgrids):
                         sg_header = s.read_subgrid_header()
                         data = s.read_subgrid_data(sg_header)
                         total = total + data.sum()
                         if i == 1:
-                            self.assertEqual(1, sg_header.get('ix'))
-                            self.assertEqual(0, sg_header.get('iy'))
-                            self.assertEqual(0, sg_header.get('iz'))
-                            self.assertEqual(6, sg_header.get('nx'))
-                            self.assertEqual(10, sg_header.get('ny'))
-                            self.assertEqual(21, sg_header.get('nz'))
+                            self.assertEqual(1, sg_header.get("ix"))
+                            self.assertEqual(0, sg_header.get("iy"))
+                            self.assertEqual(0, sg_header.get("iz"))
+                            self.assertEqual(6, sg_header.get("nx"))
+                            self.assertEqual(10, sg_header.get("ny"))
+                            self.assertEqual(21, sg_header.get("nz"))
                 self.assertEqual(-444360000.0, int(total))
 
     def test_local(self):
@@ -99,27 +94,27 @@ def test_local(self):
         da[:, 0, :] += 1
         da[:, :, 0] += 1
         header = {}
-        header['nx'] = 4
-        header['ny'] = 4
-        header['nz'] = 1
-        header['p'] = 2
-        header['q'] = 2
-        header['r'] = 1
+        header["nx"] = 4
+        header["ny"] = 4
+        header["nz"] = 1
+        header["p"] = 2
+        header["q"] = 2
+        header["r"] = 1
         with tempfile.TemporaryDirectory() as TEMP_DIRECTORY:
-            file_name = f'{TEMP_DIRECTORY}/local.pfb'
+            file_name = f"{TEMP_DIRECTORY}/local.pfb"
             write_pfb(file_name, da, **header, dist=True)
             with ParflowBinaryReader(file_name) as s:
                 # Check the header
                 header = s.header
-                self.assertEqual(4, header.get('nx'))
-                self.assertEqual(4, header.get('ny'))
-                self.assertEqual(1, header.get('nz'))
+                self.assertEqual(4, header.get("nx"))
+                self.assertEqual(4, header.get("ny"))
+                self.assertEqual(1, header.get("nz"))
 
                 # Check the subgrid header
                 sg_header = s.read_subgrid_header()
-                self.assertEqual(2, sg_header.get('nx'))
-                self.assertEqual(2, sg_header.get('ny'))
-                self.assertEqual(1, sg_header.get('nz'))
+                self.assertEqual(2, sg_header.get("nx"))
+                self.assertEqual(2, sg_header.get("ny"))
+                self.assertEqual(1, sg_header.get("nz"))
 
                 # Check the subgrid data
                 data = s.iloc_subgrid(0)
@@ -130,25 +125,25 @@ def test_local(self):
                 data = s.read_all_subgrids()
                 self.assertEqual(0, np.sum(da - data))
 
-
     def test_empty_array(self):
         da = np.arange(0, 0, dtype=np.float64)
         da.resize([0, 0, 0])
         header = {}
-        header['p'] = 1
-        header['q'] = 1
-        header['r'] = 1
+        header["p"] = 1
+        header["q"] = 1
+        header["r"] = 1
         with tempfile.TemporaryDirectory() as TEMP_DIRECTORY:
-            file_name = f'{TEMP_DIRECTORY}/empty.pfb'
+            file_name = f"{TEMP_DIRECTORY}/empty.pfb"
             write_pfb(file_name, da, **header)
             with PFBSummary() as s:
                 header = s.open_pfb(file_name)
-                self.assertEqual(0, header.get('nx'))
-                self.assertEqual(0, header.get('ny'))
-                self.assertEqual(0, header.get('nz'))
-                self.assertEqual(0, header.get('nz'))
-                number_of_subgrids = int(header.get('n_subgrids', 0))
+                self.assertEqual(0, header.get("nx"))
+                self.assertEqual(0, header.get("ny"))
+                self.assertEqual(0, header.get("nz"))
+                self.assertEqual(0, header.get("nz"))
+                number_of_subgrids = int(header.get("n_subgrids", 0))
                 self.assertEqual(1, number_of_subgrids)
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     unittest.main()
diff --git a/pftools/python/parflow/tools/top.py b/pftools/python/parflow/tools/top.py
new file mode 100644
index 000000000..d0cdd9765
--- /dev/null
+++ b/pftools/python/parflow/tools/top.py
@@ -0,0 +1,62 @@
+import numpy as np
+
+
+def compute_top(mask):
+    """Python version of the C ComputeTop function.
+
+    Computes the top indices of the computation domain as defined by
+    the mask values. Mask has values 0 outside of domain so first
+    non-zero entry is the top.
+
+    Args:
+        mask (numpy.ndarray): mask for the computation domain with shape (nz, nx, ny)
+
+    Returns:
+        A new numpy ndarray with dimensions (1, mask.shape[0], mask.shape[1])
+        with (z) indices of the top surface for each i, j location.
+    """
+    nz, nx, ny = mask.shape
+    top = np.ndarray((1, nx, ny))
+
+    for j in range(ny):
+        for i in range(nx):
+            k = nz - 1
+            while k >= 0:
+                if mask[k, i, j] > 0.0:
+                    break
+                k -= 1
+            top[0, i, j] = k
+
+    return top
+
+
+def extract_top(data, top):
+    """Python version of the C ExtractTop function.
+
+    Extracts the top values of a dataset based on a top dataset
+    (which contains the z indices that define the top of the domain).
+
+    Args:
+        data (numpy.ndarray): array of data values with shape (nz, nx, ny)
+        top (numpy.ndarray): array of z indices for each i, j location with shape (1, nx, ny)
+
+    Returns:
+        Returns a ndarray with top values extracted for each i,j location.
+
+    Raises:
+        ValueError: If a z index in top is outside the range of nz.
+    """
+    nz, nx, ny = data.shape
+    top_values_of_data = np.ndarray((1, nx, ny))
+
+    for j in range(ny):
+        for i in range(nx):
+            k = int(top[0, i, j])
+            if k < 0:
+                top_values_of_data[0, i, j] = 0.0
+            elif k < nz:
+                top_values_of_data[0, i, j] = data[k, i, j]
+            else:
+                raise ValueError(f"Index in top (k={k}) is outside of data (nz={nz})")
+
+    return top_values_of_data
diff --git a/pftools/python/parflow/tools/util.py b/pftools/python/parflow/tools/util.py
index 15a7f1bce..1f7c784bc 100644
--- a/pftools/python/parflow/tools/util.py
+++ b/pftools/python/parflow/tools/util.py
@@ -1,6 +1,7 @@
 import numpy as np
 from typing import Iterable, Union
 
+
 def _check_key_is_empty(key: slice) -> bool:
     """
     Checks if an accessor key is empty.
@@ -12,9 +13,7 @@ def _check_key_is_empty(key: slice) -> bool:
         A boolean specifying whether the key is empty
     """
     for k in key:
-        all_none = np.all([k.start is None,
-                           k.stop is None,
-                           k.step is None])
+        all_none = np.all([k.start is None, k.stop is None, k.step is None])
         if all_none:
             return True
     return False
@@ -34,29 +33,27 @@ def _key_to_explicit_accessor(key: Union[slice, int, Iterable]) -> dict:
     """
     if isinstance(key, slice):
         start = key.start if key.start is not None else 0
-        stop = key.stop+1 if key.stop is not None else -1
+        stop = key.stop + 1 if key.stop is not None else -1
         needs_squeeze = (stop - start) == 1
         accessor = {
-            'start': key.start,
-            'stop': key.stop,
-            'indices': slice(None, None, key.step),
-            'squeeze': needs_squeeze
+            "start": key.start,
+            "stop": key.stop,
+            "indices": slice(None, None, key.step),
+            "squeeze": needs_squeeze,
         }
     elif isinstance(key, int):
         accessor = {
-            'start': key,
-            'stop': key+1,
-            'indices': slice(0, 1),
-            'squeeze':  True
+            "start": key,
+            "stop": key + 1,
+            "indices": slice(0, 1),
+            "squeeze": True,
         }
     elif isinstance(key, Iterable):
         sl = int(key - np.min(key))
         accessor = {
-            'start': int(np.min(key)),
-            'stop': int(np.max(key)+1),
-            'indices': slice(sl, sl+1),
-            'squeeze': False
+            "start": int(np.min(key)),
+            "stop": int(np.max(key) + 1),
+            "indices": slice(sl, sl + 1),
+            "squeeze": False,
         }
     return accessor
-
-
diff --git a/pftools/python/pfpython b/pftools/python/pfpython
index 9f8eb23f5..84d9a5d1c 100644
--- a/pftools/python/pfpython
+++ b/pftools/python/pfpython
@@ -1,7 +1,14 @@
 #!/bin/bash
-export PARFLOW_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
+
+if [ -z ${PARFLOW_DIR+x} ]
+then
+   echo "ERROR: PARFLOW_DIR is not set"
+fi
+			 
+[ ! -d "$PARFLOW_DIR" ] && echo "ERROR: ParFlow not installed correctly or PARFLOW_DIR is not set corretly.  ParFlow not found at $PARFLOW_DIR"
+
 VIRTUAL_ENV="$PARFLOW_DIR/py-env"
 
-[ ! -d "$VIRTUAL_ENV" ] && $PARFLOW_DIR/bin/py-setup
+[ ! -d "$VIRTUAL_ENV" ] && echo "ERROR: ParFlow not installed correctly or PARFLOW_DIR is not set corretly.  Python environment not found at $VIRTUAL_ENV"
 
 $VIRTUAL_ENV/bin/python $@
diff --git a/pftools/python/py-setup b/pftools/python/py-setup
deleted file mode 100755
index fd661017a..000000000
--- a/pftools/python/py-setup
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-PARFLOW_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
-
-rm -rf "$PARFLOW_DIR/py-env"
-python3 -m venv "$PARFLOW_DIR/py-env"
-"$PARFLOW_DIR/py-env/bin/pip3" install --upgrade pip wheel setuptools
-"$PARFLOW_DIR/py-env/bin/pip3" install -r "$PARFLOW_DIR/python/requirements_all.txt"
-"$PARFLOW_DIR/py-env/bin/pip3" install "$PARFLOW_DIR/python"
diff --git a/pftools/python/requirements_dev.txt b/pftools/python/requirements_dev.txt
index 64b3d2418..46c7b2ef5 100644
--- a/pftools/python/requirements_dev.txt
+++ b/pftools/python/requirements_dev.txt
@@ -1,2 +1,3 @@
 ###### Requirements for development of PFTools package ######
 twine
+black
diff --git a/pftools/python/setup.py b/pftools/python/setup.py
index f8ce6f729..df74bc025 100644
--- a/pftools/python/setup.py
+++ b/pftools/python/setup.py
@@ -6,48 +6,47 @@
 README = (HERE / "README.md").read_text()
 
 setup(
-    name='pftools',
-    version="1.3.9",
-    description=('A Python package creating an interface with the ParFlow '
-                 'hydrologic model.'),
+    name="pftools",
+    version="1.3.11",
+    description=(
+        "A Python package creating an interface with the ParFlow " "hydrologic model."
+    ),
     long_description=README,
-    long_description_content_type='text/markdown',
-    url='https://github.com/parflow/parflow/tree/master/pftools/python',
-    author='HydroFrame',
-    author_email='parflow@parflow.org',
-    license='BSD',
+    long_description_content_type="text/markdown",
+    url="https://github.com/parflow/parflow/tree/master/pftools/python",
+    author="HydroFrame",
+    author_email="parflow@parflow.org",
+    license="BSD",
     classifiers=[
-        'Programming Language :: Python :: 3',
+        "Programming Language :: Python :: 3",
     ],
-    keywords=['ParFlow', 'groundwater model', 'surface water model'],
+    keywords=["ParFlow", "groundwater model", "surface water model"],
     packages=find_packages(),
     install_requires=[
-        'pyyaml>=5.4',
+        "pyyaml>=5.4",
     ],
     include_package_data=True,
     extras_require={
-        'all': [
-            'imageio>=2.9.0',
-            'numpy',
-            'xarray',
-            'numba',
-            'dask',
+        "all": [
+            "imageio>=2.9.0",
+            "numpy",
+            "xarray",
+            "numba",
+            "dask",
         ],
-        'pfsol': [
-            'imageio>=2.9.0'
+        "pfsol": ["imageio>=2.9.0"],
+        "io": [
+            "numpy",
+            "xarray",
+            "dask",
         ],
-        'io':[
-            'numpy',
-            'xarray',
-            'dask',
-        ],
-        'fastio': [
-            'numba',
+        "fastio": [
+            "numba",
         ],
     },
     entry_points={
-        'xarray.backends': [
-            'parflow=parflow.tools.pf_backend:ParflowBackendEntrypoint'
+        "xarray.backends": [
+            "parflow=parflow.tools.pf_backend:ParflowBackendEntrypoint"
         ],
-    }
+    },
 )
diff --git a/pftools/quicksort.c b/pftools/quicksort.c
index 67f16dbe1..9a272d30e 100644
--- a/pftools/quicksort.c
+++ b/pftools/quicksort.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*--------------------------------------------------------------------------
  * The following type must be defined to compile QuickSort:
diff --git a/pftools/readdatabox.c b/pftools/readdatabox.c
index aef49477f..87bede698 100644
--- a/pftools/readdatabox.c
+++ b/pftools/readdatabox.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Read routines for pftools.
 *
diff --git a/pftools/readdatabox.h b/pftools/readdatabox.h
index 7f59b4824..1ce846dbb 100644
--- a/pftools/readdatabox.h
+++ b/pftools/readdatabox.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Header file for `readdatabox.c'
 *
diff --git a/pftools/region.c b/pftools/region.c
index 6e2b0852a..b6e99c9f1 100644
--- a/pftools/region.c
+++ b/pftools/region.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include <math.h>
 #include "pfload_file.h"
diff --git a/pftools/region.h b/pftools/region.h
index 1d32d0055..9b41eada6 100644
--- a/pftools/region.h
+++ b/pftools/region.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef _REGION_HEADER
 #define _REGION_HEADER
@@ -132,7 +132,7 @@ typedef struct {
  *--------------------------------------------------------------------------*/
 
 #define SubregionArraySubregion(subregion_array, i) \
-  ((subregion_array)->subregions[(i)])
+        ((subregion_array)->subregions[(i)])
 #define SubregionArraySize(subregion_array)  ((subregion_array)->size)
 
 /*--------------------------------------------------------------------------
@@ -147,10 +147,10 @@ typedef struct {
  *--------------------------------------------------------------------------*/
 
 #define ForSubregionI(i, subregion_array) \
-  for (i = 0; i < SubregionArraySize(subregion_array); i ++)
+        for (i = 0; i < SubregionArraySize(subregion_array); i++)
 
 #define ForSubregionArrayI(i, region) \
-  for (i = 0; i < RegionSize(region); i ++)
+        for (i = 0; i < RegionSize(region); i++)
 
 
 /* region.c */
diff --git a/pftools/slimtopfsb.c b/pftools/slimtopfsb.c
index 706c51a93..8ffc7413d 100644
--- a/pftools/slimtopfsb.c
+++ b/pftools/slimtopfsb.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include <stdio.h>
 
 int main(int argc, char *argv[])
diff --git a/pftools/solidtools.c b/pftools/solidtools.c
index 088078e2e..4bc0082e2 100644
--- a/pftools/solidtools.c
+++ b/pftools/solidtools.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 // - Holds a variety of utilities for building and manipulating solid files
 
@@ -53,22 +53,23 @@ typedef struct {
 } PatchInfo;
 
 /*-----------------------------------------------------------------------
-* Returns index of array "in_list[len]" that holds "val" if it exists, -1 otherwise
------------------------------------------------------------------------*/
+ * Returns index of array "in_list[len]" that holds "val" if it exists, -1 otherwise
+ *  -----------------------------------------------------------------------*/
 int ScanArray(int val, int in_list[], int len) //
+{
+  int n = -1;
+  int i;
+
+  for (i = 0; i < len; ++i)
   {
-   int n=-1;
-   int i;
-   for (i=0; i< len; ++i)
-   {
-     if (in_list[i]==val)
-     {
-       n=i;
-       break;
-     }
-   }
-  return n;
+    if (in_list[i] == val)
+    {
+      n = i;
+      break;
+    }
   }
+  return n;
+}
 
 /*-----------------------------------------------------------------------
  * Write a complex solid file with patches requires a mask that is an indicator
@@ -79,16 +80,16 @@ int ScanArray(int val, int in_list[], int len) //
  *
  *-----------------------------------------------------------------------*/
 int            MakePatchySolid(
-                             FILE *   fp,
-                             FILE *   fp_vtk,
-                             Databox *msk,
-                             Databox *top,
-                             Databox *bot,
-                             int sub_patches,
-                             int bin_out)
+                               FILE *   fp,
+                               FILE *   fp_vtk,
+                               Databox *msk,
+                               Databox *top,
+                               Databox *bot,
+                               int      sub_patches,
+                               int      bin_out)
 {
-  int out_status=-1;
-  int i,j,k,m;
+  int out_status = -1;
+  int i, j, k, m;
   int NX = DataboxNx(msk);
   int NY = DataboxNy(msk);
   int NZ = DataboxNz(msk);
@@ -98,31 +99,32 @@ int            MakePatchySolid(
   double DY = DataboxDy(msk);
 
   /* ---------- Max number of patches ---------- */
-  int np_tot=30;        // Max number of patches allowed (CAN BE INCREASED IF NEEDED)
+  int np_tot = 30;        // Max number of patches allowed (CAN BE INCREASED IF NEEDED)
 
-  int debugger=0;
+  int debugger = 0;
 
   // #include "time.h"
   // clock_t start_time, end_time;
   // double run_time;
 
   // Default patches (flagged as negatives here)
-  int DefPatches[] = {-1,-2,-3,-4,-5,-6}; // Bottom, Top, West, East, South, North
-  int DefPCounts[]={0,0,0,0,0,0};   // Count for each face patch
+  int DefPatches[] = { -1, -2, -3, -4, -5, -6 }; // Bottom, Top, West, East, South, North
+  int DefPCounts[] = { 0, 0, 0, 0, 0, 0 };   // Count for each face patch
 
   int UsrPatches[np_tot];           // User defined patches, up to np_tot
   int UsrPCounts[np_tot];           // Counts for number of patches on user patches
-  int np_usr=-1;                    // Counter for total user patch
+  int np_usr = -1;                    // Counter for total user patch
+
   for (i = 0; i < np_tot; ++i)
   {
-    UsrPatches[i]=-99999; // Initialize to a BIG negative
-    UsrPCounts[i]=0;
+    UsrPatches[i] = -99999; // Initialize to a BIG negative
+    UsrPCounts[i] = 0;
   }
 
-  if (NZ!=1)
+  if (NZ != 1)
   {
     printf("\n ERROR (pfpatchysolid): 2-d input required for mask dataset\n");
-    out_status=-2;
+    out_status = -2;
     return out_status;
   }
 
@@ -134,228 +136,271 @@ int            MakePatchySolid(
   int NYb = DataboxNy(bot);
   int NZb = DataboxNz(bot);
 
-  if ((NX!=NXt)||(NX!=NXb)||(NY!=NYt)||(NY!=NYb)||(NZ!=NZt)||(NZ!=NZb))
+  if ((NX != NXt) || (NX != NXb) || (NY != NYt) || (NY != NYb) || (NZ != NZt) || (NZ != NZb))
   {
     printf("\n ERROR (pfpatchysolid): Inconsistent input dataset dimensions\n");
-    out_status=-2;
+    out_status = -2;
     return out_status;
   }
 
   int nxyzp = (NX + 1) * (NY + 1); // NOTE: Z is NOT included here
 
-  int any_zeros=0;
+  int any_zeros = 0;
 
   // Find the distinct patches in the mask and tally them up
-  int mask_val=0;
-  int test_val=0;
-  int idx=0;
+  int mask_val = 0;
+  int test_val = 0;
+  int idx = 0;
   for (j = 0; j < NY; ++j)
   {
     for (i = 0; i < NX; ++i)
     {
       mask_val = *DataboxCoeff(msk, i, j, 0);
 
-      if ( (any_zeros==0) && (mask_val==0))
+      if ((any_zeros == 0) && (mask_val == 0))
       {
-        any_zeros=1;
+        any_zeros = 1;
       }
 
-      if (mask_val==1)
+      if (mask_val == 1)
       {
-        DefPCounts[0]=DefPCounts[0]+1;  // Increment BOTTOM and TOP counts
-        DefPCounts[1]=DefPCounts[1]+1;
+        DefPCounts[0] = DefPCounts[0] + 1;  // Increment BOTTOM and TOP counts
+        DefPCounts[1] = DefPCounts[1] + 1;
         // Check for edge faces
-        if (i==0) {DefPCounts[2]=DefPCounts[2]+1;}
-        if (i==(NX-1)) {DefPCounts[3]=DefPCounts[3]+1;}
-        if (j==0) {DefPCounts[4]=DefPCounts[4]+1;}
-        if (j==(NY-1)) {DefPCounts[5]=DefPCounts[5]+1;}
+        if (i == 0)
+        {
+          DefPCounts[2] = DefPCounts[2] + 1;
+        }
+        if (i == (NX - 1))
+        {
+          DefPCounts[3] = DefPCounts[3] + 1;
+        }
+        if (j == 0)
+        {
+          DefPCounts[4] = DefPCounts[4] + 1;
+        }
+        if (j == (NY - 1))
+        {
+          DefPCounts[5] = DefPCounts[5] + 1;
+        }
 
         // Now check the four neighbors
-        if (i>0) // Left neighbor (WEST)
+        if (i > 0) // Left neighbor (WEST)
         {
-          test_val = *DataboxCoeff(msk, i-1, j, 0);
-          if (test_val!=1)
+          test_val = *DataboxCoeff(msk, i - 1, j, 0);
+          if (test_val != 1)
           {
             // Find its index and add to the count, or create a new patch
-            idx=ScanArray(test_val,UsrPatches,np_tot);
-              if (idx<0)
-              {
-                np_usr=np_usr+1; // Add the integer to the list, augment the count
-                if (np_usr>np_tot) {printf("ERROR: Too many patches\n"); out_status=-2; return out_status;}
-                UsrPatches[np_usr]=test_val;
-                UsrPCounts[np_usr]=UsrPCounts[np_usr]+1;
-              }
-              else
+            idx = ScanArray(test_val, UsrPatches, np_tot);
+            if (idx < 0)
+            {
+              np_usr = np_usr + 1; // Add the integer to the list, augment the count
+              if (np_usr > np_tot)
               {
-                UsrPCounts[idx]=UsrPCounts[idx]+1;
+                printf("ERROR: Too many patches\n"); out_status = -2; return out_status;
               }
+              UsrPatches[np_usr] = test_val;
+              UsrPCounts[np_usr] = UsrPCounts[np_usr] + 1;
+            }
+            else
+            {
+              UsrPCounts[idx] = UsrPCounts[idx] + 1;
+            }
           }
         } // End of WEST
-        if (i<(NX-1)) // Right neighbor (EAST)
+        if (i < (NX - 1)) // Right neighbor (EAST)
         {
-          test_val = *DataboxCoeff(msk, i+1, j, 0);
-          if (test_val!=1)
+          test_val = *DataboxCoeff(msk, i + 1, j, 0);
+          if (test_val != 1)
           {
             // Find its index and add to the count, or create a new patch
-            idx=ScanArray(test_val,UsrPatches,np_tot);
-              if (idx<0)
-              {
-                np_usr=np_usr+1; // Add the integer to the list, augment the count
-                if (np_usr>np_tot) {printf("ERROR: Too many patches\n"); out_status=-2; return out_status;}
-                UsrPatches[np_usr]=test_val;
-                UsrPCounts[np_usr]=UsrPCounts[np_usr]+1;
-              } else
+            idx = ScanArray(test_val, UsrPatches, np_tot);
+            if (idx < 0)
+            {
+              np_usr = np_usr + 1; // Add the integer to the list, augment the count
+              if (np_usr > np_tot)
               {
-                UsrPCounts[idx]=UsrPCounts[idx]+1;
+                printf("ERROR: Too many patches\n"); out_status = -2; return out_status;
               }
+              UsrPatches[np_usr] = test_val;
+              UsrPCounts[np_usr] = UsrPCounts[np_usr] + 1;
+            }
+            else
+            {
+              UsrPCounts[idx] = UsrPCounts[idx] + 1;
+            }
           }
         } // End of EAST
-        if (j>0) // Lower neighbor (SOUTH)
+        if (j > 0) // Lower neighbor (SOUTH)
         {
-          test_val = *DataboxCoeff(msk, i, j-1, 0);
-          if (test_val!=1)
+          test_val = *DataboxCoeff(msk, i, j - 1, 0);
+          if (test_val != 1)
           {
             // Find its index and add to the count, or create a new patch
-            idx=ScanArray(test_val,UsrPatches,np_tot);
-              if (idx<0)
-              {
-                np_usr=np_usr+1; // Add the integer to the list, augment the count
-                if (np_usr>np_tot) {printf("ERROR: Too many patches\n"); out_status=-2; return out_status;}
-                UsrPatches[np_usr]=test_val;
-                UsrPCounts[np_usr]=UsrPCounts[np_usr]+1;
-              } else
+            idx = ScanArray(test_val, UsrPatches, np_tot);
+            if (idx < 0)
+            {
+              np_usr = np_usr + 1; // Add the integer to the list, augment the count
+              if (np_usr > np_tot)
               {
-                UsrPCounts[idx]=UsrPCounts[idx]+1;
+                printf("ERROR: Too many patches\n"); out_status = -2; return out_status;
               }
+              UsrPatches[np_usr] = test_val;
+              UsrPCounts[np_usr] = UsrPCounts[np_usr] + 1;
+            }
+            else
+            {
+              UsrPCounts[idx] = UsrPCounts[idx] + 1;
+            }
           }
         } // End of SOUTH
-        if (j<(NY-1)) // Upper neighbor (NORTH)
+        if (j < (NY - 1)) // Upper neighbor (NORTH)
         {
-          test_val = *DataboxCoeff(msk, i, j+1, 0);
-          if (test_val!=1)
+          test_val = *DataboxCoeff(msk, i, j + 1, 0);
+          if (test_val != 1)
           {
             // Find its index and add to the count, or create a new patch
-            idx=ScanArray(test_val,UsrPatches,np_tot);
-              if (idx<0)
-              {
-                np_usr=np_usr+1; // Add the integer to the list, augment the count
-                if (np_usr>np_tot) {printf("ERROR: Too many patches\n"); out_status=-2; return out_status;}
-                UsrPatches[np_usr]=test_val;
-                UsrPCounts[np_usr]=UsrPCounts[np_usr]+1;
-              } else
+            idx = ScanArray(test_val, UsrPatches, np_tot);
+            if (idx < 0)
+            {
+              np_usr = np_usr + 1; // Add the integer to the list, augment the count
+              if (np_usr > np_tot)
               {
-                UsrPCounts[idx]=UsrPCounts[idx]+1;
+                printf("ERROR: Too many patches\n"); out_status = -2; return out_status;
               }
+              UsrPatches[np_usr] = test_val;
+              UsrPCounts[np_usr] = UsrPCounts[np_usr] + 1;
+            }
+            else
+            {
+              UsrPCounts[idx] = UsrPCounts[idx] + 1;
+            }
           }
         } // End of NORTH
       } // end of mask_val test
     } // end of j loop
   } // end of i loop
 
-  int ix_off[4],iy_off[4],jx_off[4],jy_off[4];
-  int off_ref[]={0,-1,1,0};
+  int ix_off[4], iy_off[4], jx_off[4], jy_off[4];
+  int off_ref[] = { 0, -1, 1, 0 };
 
-  double x_test,y_test;
+  double x_test, y_test;
   double z_bot;
-  double dx_b,dy_b;
+  double dx_b, dy_b;
 
   // =========================================================================
   // Sort the patches in ASCENDING order to make life easier
 
-  int TempUPCnt[np_usr+1],TempUPLst[np_usr+1];
-    for (i=0; i<(np_usr+1); ++i)
+  int TempUPCnt[np_usr + 1], TempUPLst[np_usr + 1];
+  for (i = 0; i < (np_usr + 1); ++i)
   {
     // printf(" %i %i \n",UsrPatches[i],UsrPCounts[i]);
-    TempUPCnt[i]=UsrPCounts[i];
-    TempUPLst[i]=UsrPatches[i];
-    UsrPCounts[i]=-1;       // Reset so they can be sorted
-    UsrPatches[i]=-99999;
+    TempUPCnt[i] = UsrPCounts[i];
+    TempUPLst[i] = UsrPatches[i];
+    UsrPCounts[i] = -1;       // Reset so they can be sorted
+    UsrPatches[i] = -99999;
   }
   // Find the minimum
 
-  int min_val,min_idx;
-  for (j=0; j<(np_usr+1); ++j)
+  int min_val, min_idx;
+  for (j = 0; j < (np_usr + 1); ++j)
   {
-      min_val=TempUPLst[j];
-      min_idx=j;
-    for (k=(j+1); k<(np_usr+1); ++k)
+    min_val = TempUPLst[j];
+    min_idx = j;
+    for (k = (j + 1); k < (np_usr + 1); ++k)
     {
-      if (TempUPLst[k]<min_val)
+      if (TempUPLst[k] < min_val)
       {
-        min_val=TempUPLst[k];
-        min_idx=k;
+        min_val = TempUPLst[k];
+        min_idx = k;
       }
     } // k loop end
-  UsrPCounts[j]=TempUPCnt[min_idx];
-  UsrPatches[j]=TempUPLst[min_idx];
-  // Now swap the values
-  TempUPLst[min_idx]=TempUPLst[j]; TempUPLst[j]=UsrPatches[j];
-  TempUPCnt[min_idx]=TempUPCnt[j]; TempUPCnt[j]=UsrPCounts[j];
+    UsrPCounts[j] = TempUPCnt[min_idx];
+    UsrPatches[j] = TempUPLst[min_idx];
+    // Now swap the values
+    TempUPLst[min_idx] = TempUPLst[j]; TempUPLst[j] = UsrPatches[j];
+    TempUPCnt[min_idx] = TempUPCnt[j]; TempUPCnt[j] = UsrPCounts[j];
   }
 
   // Now move on to building the solids and the patches
-  int np=0;
-  int cell_faces=0;
+  int np = 0;
+  int cell_faces = 0;
   // Count up all the cell faces
 
-  for (i=0; i<6; ++i) {cell_faces=cell_faces+DefPCounts[i];}
-  for (i=0; i<np_tot; ++i)
+  for (i = 0; i < 6; ++i)
+  {
+    cell_faces = cell_faces + DefPCounts[i];
+  }
+  for (i = 0; i < np_tot; ++i)
   {
-    cell_faces=cell_faces+UsrPCounts[i];
+    cell_faces = cell_faces + UsrPCounts[i];
   }
 
   // Create the array structure to hold the info for those patches
-  PatchInfo AllPatches[6+np_usr];
-  int non_blanks=0;
+  PatchInfo AllPatches[6 + np_usr];
+  int non_blanks = 0;
 
-  np=0;
-  for (i=0; i<6; ++i)
+  np = 0;
+  for (i = 0; i < 6; ++i)
   {
-    if (DefPCounts[i]>0)
+    if (DefPCounts[i] > 0)
     {
-      AllPatches[i].p_write=-1;
-      AllPatches[i].active=1;
-      AllPatches[i].value=DefPatches[i];
-      AllPatches[i].patch_cell_count=DefPCounts[i];
-      AllPatches[i].start_idx=np;
-      AllPatches[i].end_idx=np+AllPatches[i].patch_cell_count-1;
-      np=AllPatches[i].end_idx+1;
-      non_blanks=non_blanks+1;
-      for (j=0; j<6; ++j) {AllPatches[i].act_faces[j]=0;} // Bottom, Top, West, East, South, North
-      AllPatches[i].total_patches=0;
-    } else {
-      AllPatches[i].p_write=-1;
-      AllPatches[i].active=0;
-      AllPatches[i].value=DefPatches[i];
-      AllPatches[i].patch_cell_count=0;
-      AllPatches[i].start_idx=-1;
-      AllPatches[i].end_idx=-1;
-      for (j=0; j<6; ++j) {AllPatches[i].act_faces[j]=0;} // Bottom, Top, West, East, South, North
+      AllPatches[i].p_write = -1;
+      AllPatches[i].active = 1;
+      AllPatches[i].value = DefPatches[i];
+      AllPatches[i].patch_cell_count = DefPCounts[i];
+      AllPatches[i].start_idx = np;
+      AllPatches[i].end_idx = np + AllPatches[i].patch_cell_count - 1;
+      np = AllPatches[i].end_idx + 1;
+      non_blanks = non_blanks + 1;
+      for (j = 0; j < 6; ++j)
+      {
+        AllPatches[i].act_faces[j] = 0;
+      }                                                   // Bottom, Top, West, East, South, North
+      AllPatches[i].total_patches = 0;
+    }
+    else
+    {
+      AllPatches[i].p_write = -1;
+      AllPatches[i].active = 0;
+      AllPatches[i].value = DefPatches[i];
+      AllPatches[i].patch_cell_count = 0;
+      AllPatches[i].start_idx = -1;
+      AllPatches[i].end_idx = -1;
+      for (j = 0; j < 6; ++j)
+      {
+        AllPatches[i].act_faces[j] = 0;
+      }                                                   // Bottom, Top, West, East, South, North
     }
   }
-  for (i=6; i<(6+np_usr+1); ++i)
+  for (i = 6; i < (6 + np_usr + 1); ++i)
   {
-    if (UsrPCounts[i-6]>0)
+    if (UsrPCounts[i - 6] > 0)
+    {
+      AllPatches[i].p_write = -1;
+      AllPatches[i].active = 1;
+      AllPatches[i].value = UsrPatches[i - 6];
+      AllPatches[i].patch_cell_count = UsrPCounts[i - 6];
+      AllPatches[i].start_idx = np;
+      AllPatches[i].end_idx = np + AllPatches[i].patch_cell_count - 1;
+      np = AllPatches[i].end_idx + 1;
+      non_blanks = non_blanks + 1;
+      for (j = 0; j < 6; ++j)
+      {
+        AllPatches[i].act_faces[j] = 0;
+      }                                                   // Bottom, Top, West, East, South, North
+      AllPatches[i].total_patches = 0;
+    }
+    else
     {
-      AllPatches[i].p_write=-1;
-      AllPatches[i].active=1;
-      AllPatches[i].value=UsrPatches[i-6];
-      AllPatches[i].patch_cell_count=UsrPCounts[i-6];
-      AllPatches[i].start_idx=np;
-      AllPatches[i].end_idx=np+AllPatches[i].patch_cell_count-1;
-      np=AllPatches[i].end_idx+1;
-      non_blanks=non_blanks+1;
-      for (j=0; j<6; ++j) {AllPatches[i].act_faces[j]=0;} // Bottom, Top, West, East, South, North
-      AllPatches[i].total_patches=0;
-    } else {
       // AllPatches[i].active=0;
       // AllPatches[i].patch_cell_count=0;
       // AllPatches[i].value=-1;
       // AllPatches[i].start_idx=-1;
       // AllPatches[i].end_idx=-1;
       printf("\n ERROR: Patch indexing failure...MUST STOP\n");
-      printf("    Value if: %i (%i) \n",i-6,i);
-      printf("Pcount, Pval: %i %i\n",UsrPCounts[i-6],UsrPatches[i-6]);
+      printf("    Value if: %i (%i) \n", i - 6, i);
+      printf("Pcount, Pval: %i %i\n", UsrPCounts[i - 6], UsrPatches[i - 6]);
       return out_status;
     }
   }
@@ -368,366 +413,447 @@ int            MakePatchySolid(
   // }
 
   // Used to easily find the ID of user patches and zeros later
-  int patch_values[6+np_usr];
-  for (i=0; i<(6+np_usr+1); ++i)
+  int patch_values[6 + np_usr];
+  for (i = 0; i < (6 + np_usr + 1); ++i)
   {
-    patch_values[i]=AllPatches[i].value;
+    patch_values[i] = AllPatches[i].value;
   }
 
   // int Patch[cell_faces*7];
   int *Patch;
   // Patch=(int*)calloc(cell_faces*7,sizeof(int));
-  Patch=(int*)calloc(cell_faces*8,sizeof(int));
+  Patch = (int*)calloc(cell_faces * 8, sizeof(int));
 
   // for (i=0;i<(cell_faces*7); i++) {Patch[i]=-1111;}
-  for (i=0;i<(cell_faces*8); i++) {Patch[i]=-1111;}
+  for (i = 0; i < (cell_faces * 8); i++)
+  {
+    Patch[i] = -1111;
+  }
 
-  if (debugger==1)
+  if (debugger == 1)
   {
-    printf(" *** Cell faces: %i *** \n",cell_faces);
-    printf("Should be: %i\n",NX*2 + NY*2 + (NX*NY)*2);
-    printf("NX,NY: %i %i\n",NX, NY);
+    printf(" *** Cell faces: %i *** \n", cell_faces);
+    printf("Should be: %i\n", NX * 2 + NY * 2 + (NX * NY) * 2);
+    printf("NX,NY: %i %i\n", NX, NY);
 
-    printf("any_zeros value: %i\n",any_zeros);
+    printf("any_zeros value: %i\n", any_zeros);
   }
 
   // ==========================================================================
   // Build a 2-d array to hold the active points
 
-  if (debugger==1) printf("NXY points: %i\n",nxyzp);
+  if (debugger == 1)
+    printf("NXY points: %i\n", nxyzp);
 
   // double Xp_Act[nxyzp*6];
   double *Xp_Act;
-  Xp_Act=(double*)calloc(nxyzp*6,sizeof(double));
+  Xp_Act = (double*)calloc(nxyzp * 6, sizeof(double));
 
-  for (i=0;i<nxyzp; i++) {for (j=0;j<3; j++) {Xp_Act[3*i + j]=-11111.11;}}
+  for (i = 0; i < nxyzp; i++)
+  {
+    for (j = 0; j < 3; j++)
+    {
+      Xp_Act[3 * i + j] = -11111.11;
+    }
+  }
 
   // Which corners of the cell to add to the Xp_Act dBase, who belongs to this cell
-  int add_pnt[8],our_pnts[8];
-  int i_off=0, j_off=0;
-  int km,n_t;
-  int np_act=-1;
+  int add_pnt[8], our_pnts[8];
+  int i_off = 0, j_off = 0;
+  int km, n_t;
+  int np_act = -1;
 
   // if (debugger==1) start_time=clock(); // Enable <time.h> at top of function
 
   // Build the point database, not efficient to loop again but oh well
-  for (j=0; j<NY; ++j) //NY
-  {
-  for (i=0; i<NX; ++i) //NX
+  for (j = 0; j < NY; ++j) //NY
   {
-    // if (debugger==1) printf("Current i,j,np_act: %i, %i, %i\n",i,j,np_act);
-
-    mask_val = *DataboxCoeff(msk, i, j, 0);
-    if (mask_val==1)
+    for (i = 0; i < NX; ++i) //NX
     {
-    for (k=0;k<8;++k){add_pnt[k]=1;} // Initially assume everybody gets added
-    for (k=0;k<8;++k){our_pnts[k]=-1;}
+      // if (debugger==1) printf("Current i,j,np_act: %i, %i, %i\n",i,j,np_act);
 
-    // Determine who doesn't need to be added
-    if (i>0) {
-        if (*DataboxCoeff(msk, i-1, j, 0)==1) {
-          add_pnt[0]=0; add_pnt[2]=0; add_pnt[4]=0; add_pnt[6]=0;
+      mask_val = *DataboxCoeff(msk, i, j, 0);
+      if (mask_val == 1)
+      {
+        for (k = 0; k < 8; ++k)
+        {
+          add_pnt[k] = 1;
+        }                            // Initially assume everybody gets added
+        for (k = 0; k < 8; ++k)
+        {
+          our_pnts[k] = -1;
         }
-    }
-    if ((i>0)&(j>0)) {
-        if (*DataboxCoeff(msk, i-1, j-1, 0)==1) {
-          add_pnt[0]=0; add_pnt[4]=0;
+
+        // Determine who doesn't need to be added
+        if (i > 0)
+        {
+          if (*DataboxCoeff(msk, i - 1, j, 0) == 1)
+          {
+            add_pnt[0] = 0; add_pnt[2] = 0; add_pnt[4] = 0; add_pnt[6] = 0;
+          }
         }
-    }
-    if (j>0) {
-        if (*DataboxCoeff(msk, i, j-1, 0)==1) {
-          add_pnt[0]=0; add_pnt[1]=0; add_pnt[4]=0; add_pnt[5]=0;
+        if ((i > 0) & (j > 0))
+        {
+          if (*DataboxCoeff(msk, i - 1, j - 1, 0) == 1)
+          {
+            add_pnt[0] = 0; add_pnt[4] = 0;
+          }
         }
-    }
-    if ((i<(NX-1))&(j>0)) {
-        if (*DataboxCoeff(msk, i+1, j-1, 0)==1) {
-          add_pnt[1]=0; add_pnt[5]=0;
+        if (j > 0)
+        {
+          if (*DataboxCoeff(msk, i, j - 1, 0) == 1)
+          {
+            add_pnt[0] = 0; add_pnt[1] = 0; add_pnt[4] = 0; add_pnt[5] = 0;
+          }
+        }
+        if ((i < (NX - 1)) & (j > 0))
+        {
+          if (*DataboxCoeff(msk, i + 1, j - 1, 0) == 1)
+          {
+            add_pnt[1] = 0; add_pnt[5] = 0;
+          }
         }
-    }
 
-    // Now add the new points to Xp_Act
-    for (km=0; km<8; ++km)
-    {
-      if (add_pnt[km]==1)
-      {
-              i_off=0; j_off=0;
-              np_act=np_act+1;
-              our_pnts[km]=np_act;
-              if ((km==1)|(km==3)|(km==5)|(km==7)) {i_off=1;}
-              if ((km==2)|(km==3)|(km==6)|(km==7)) {j_off=1;}
+        // Now add the new points to Xp_Act
+        for (km = 0; km < 8; ++km)
+        {
+          if (add_pnt[km] == 1)
+          {
+            i_off = 0; j_off = 0;
+            np_act = np_act + 1;
+            our_pnts[km] = np_act;
+            if ((km == 1) | (km == 3) | (km == 5) | (km == 7))
+            {
+              i_off = 1;
+            }
+            if ((km == 2) | (km == 3) | (km == 6) | (km == 7))
+            {
+              j_off = 1;
+            }
+
+            Xp_Act[3 * np_act] = X + (i + i_off) * DX;
+            Xp_Act[3 * np_act + 1] = Y + (j + j_off) * DY;
+
+            // The simple linear interpolation
+            z_bot = -3333.33;
+            dx_b = 0.0; dy_b = 0.0;
 
-              Xp_Act[3*np_act]= X + (i+i_off)*DX;
-              Xp_Act[3*np_act+1]= Y + (j+j_off)*DY;
+            for (k = 0; k < 4; ++k)
+            {
+              ix_off[k] = off_ref[k];
+              jy_off[k] = off_ref[k];
 
-              // The simple linear interpolation
-              z_bot=-3333.33;
-              dx_b=0.0; dy_b=0.0;
+              jx_off[k] = 0;
+              iy_off[k] = 0;
+            }
 
-              for (k=0;k<4; ++k)
+            if (i == 0)
+            {
+              ix_off[0] = 1; ix_off[1] = 0;
+            }
+            else if ((i + 1) == (NX))
+            {
+              ix_off[2] = 0; ix_off[3] = -1;
+              for (k = 0; k < 4; ++k)
               {
-                ix_off[k]=off_ref[k];
-                jy_off[k]=off_ref[k];
+                iy_off[k] = -1;
+              }
+            }
 
-                jx_off[k]=0;
-                iy_off[k]=0;
+            if (j == 0)
+            {
+              jy_off[0] = 1; jy_off[1] = 0;
+            }
+            else if ((j + 1) == (NY))
+            {
+              jy_off[2] = 0; jy_off[3] = -1;
+              for (k = 0; k < 4; ++k)
+              {
+                jx_off[k] = -2;
               }
+            }
+
+            if ((km == 0) | (km == 1) | (km == 2) | (km == 3))
+            {
+              dx_b = ((*DataboxCoeff(bot, i + ix_off[0], j + jx_off[0], 0) - *DataboxCoeff(bot, i + ix_off[1], j + jx_off[1], 0)) / DX +
+                      (*DataboxCoeff(bot, i + ix_off[2], j + jx_off[2], 0) - *DataboxCoeff(bot, i + ix_off[3], j + jx_off[3], 0)) / DX) / 2.0;
+              dy_b = ((*DataboxCoeff(bot, i + iy_off[0], j + jy_off[0], 0) - *DataboxCoeff(bot, i + iy_off[1], j + jy_off[1], 0)) / DX +
+                      (*DataboxCoeff(bot, i + iy_off[2], j + jy_off[2], 0) - *DataboxCoeff(bot, i + iy_off[3], j + jy_off[3], 0)) / DX) / 2.0;
 
-              if (i==0)
+              // HERE: These need a km check instead of just an i j check
+              if ((km == 1) | (km == 3))
               {
-                ix_off[0]=1; ix_off[1]=0;
-              } else if ((i+1)==(NX))
+                dx_b = -dx_b;
+              }
+              if ((km == 2) | (km == 3))
               {
-                ix_off[2]=0; ix_off[3]=-1;
-                for (k=0; k<4; ++k) {iy_off[k]=-1;}
+                dy_b = -dy_b;
               }
+              z_bot = *DataboxCoeff(bot, i, j, 0) - dx_b * DX / 2.0 - dy_b * DY / 2.0;
+            }
+            else
+            {
+              dx_b = ((*DataboxCoeff(top, i + ix_off[0], j + jx_off[0], 0) - *DataboxCoeff(top, i + ix_off[1], j + jx_off[1], 0)) / DX +
+                      (*DataboxCoeff(top, i + ix_off[2], j + jx_off[2], 0) - *DataboxCoeff(top, i + ix_off[3], j + jx_off[3], 0)) / DX) / 2.0;
+              dy_b = ((*DataboxCoeff(top, i + iy_off[0], j + jy_off[0], 0) - *DataboxCoeff(top, i + iy_off[1], j + jy_off[1], 0)) / DX +
+                      (*DataboxCoeff(top, i + iy_off[2], j + jy_off[2], 0) - *DataboxCoeff(top, i + iy_off[3], j + jy_off[3], 0)) / DX) / 2.0;
 
-              if (j==0)
+              if ((km == 5) | (km == 7))
               {
-                jy_off[0]=1; jy_off[1]=0;
-              } else if ((j+1)==(NY))
+                dx_b = -dx_b;
+              }
+              if ((km == 6) | (km == 7))
               {
-                jy_off[2]=0; jy_off[3]=-1;
-                for (k=0; k<4; ++k) {jx_off[k]=-2;}
+                dy_b = -dy_b;
               }
+              z_bot = *DataboxCoeff(top, i, j, 0) - dx_b * DX / 2.0 - dy_b * DY / 2.0;
+            }
+            Xp_Act[3 * np_act + 2] = z_bot; // BOTTOM elevation
+          }
+          else
+          {
+            // Point already exists so go find its index
+            i_off = 0; j_off = 0;
+            if ((km == 1) | (km == 3) | (km == 5) | (km == 7))
+            {
+              i_off = 1;
+            }
+            if ((km == 2) | (km == 3) | (km == 6) | (km == 7))
+            {
+              j_off = 1;
+            }
+            x_test = X + (i + i_off) * DX;
+            y_test = Y + (j + j_off) * DY;
 
-              if ((km==0)|(km==1)|(km==2)|(km==3))
+            /* Loop last to first since points will be closer */
+            for (m = np_act; m > 0; m = m - 1)
+            {
+              if ((Xp_Act[3 * m] == x_test) & (Xp_Act[3 * m + 1] == y_test))
               {
-                dx_b=((*DataboxCoeff(bot, i+ix_off[0], j+jx_off[0], 0) - *DataboxCoeff(bot, i+ix_off[1], j+jx_off[1], 0))/DX +
-                    (*DataboxCoeff(bot, i+ix_off[2], j+jx_off[2], 0) - *DataboxCoeff(bot, i+ix_off[3], j+jx_off[3], 0))/DX)/2.0;
-                dy_b=((*DataboxCoeff(bot, i+iy_off[0], j+jy_off[0], 0) - *DataboxCoeff(bot, i+iy_off[1], j+jy_off[1], 0))/DX +
-                    (*DataboxCoeff(bot, i+iy_off[2], j+jy_off[2], 0) - *DataboxCoeff(bot, i+iy_off[3], j+jy_off[3], 0))/DX)/2.0;
-
-                    // HERE: These need a km check instead of just an i j check
-                    if ((km==1)|(km==3)) {dx_b=-dx_b;}
-                    if ((km==2)|(km==3)) {dy_b=-dy_b;}
-                    z_bot=*DataboxCoeff(bot, i, j, 0) - dx_b*DX/2.0 - dy_b*DY/2.0;
-
-              } else {
-                dx_b=((*DataboxCoeff(top, i+ix_off[0], j+jx_off[0], 0) - *DataboxCoeff(top, i+ix_off[1], j+jx_off[1], 0))/DX +
-                        (*DataboxCoeff(top, i+ix_off[2], j+jx_off[2], 0) - *DataboxCoeff(top, i+ix_off[3], j+jx_off[3], 0))/DX)/2.0;
-                dy_b=((*DataboxCoeff(top, i+iy_off[0], j+jy_off[0], 0) - *DataboxCoeff(top, i+iy_off[1], j+jy_off[1], 0))/DX +
-                        (*DataboxCoeff(top, i+iy_off[2], j+jy_off[2], 0) - *DataboxCoeff(top, i+iy_off[3], j+jy_off[3], 0))/DX)/2.0;
-
-                    if ((km==5)|(km==7)) {dx_b=-dx_b;}
-                    if ((km==6)|(km==7)) {dy_b=-dy_b;}
-                    z_bot=*DataboxCoeff(top, i, j, 0) - dx_b*DX/2.0 - dy_b*DY/2.0;
+                break;
               }
-              Xp_Act[3*np_act+2]=z_bot;   // BOTTOM elevation
-
-      } else {
-        // Point already exists so go find its index
-      i_off=0; j_off=0;
-      if ((km==1)|(km==3)|(km==5)|(km==7)) {i_off=1;}
-      if ((km==2)|(km==3)|(km==6)|(km==7)) {j_off=1;}
-      x_test = X + (i+i_off)*DX;
-      y_test = Y + (j+j_off)*DY;
-
-      /* Loop last to first since points will be closer */
-      for (m=np_act; m>0; m=m-1)
-      {
-        if ((Xp_Act[3*m]==x_test)&(Xp_Act[3*m+1]==y_test))
+            }
+            // If it's a bottom point keep going to find the next instance of the x-y coodrinates
+            if (km <= 3)
+            {
+              for (m = m - 1; m > 0; m = m - 1)
+              {
+                if ((Xp_Act[3 * m] == x_test) & (Xp_Act[3 * m + 1] == y_test))
+                {
+                  break;
+                }
+              }
+            }
+
+            our_pnts[km] = m;
+          }
+        }
+
+        // Always add a top and bottom patch
+        /* ----- BOTTOM patch ----- */
+        AllPatches[0].p_write = AllPatches[0].p_write + 1;
+        n_t = AllPatches[0].start_idx + AllPatches[0].p_write;
+        Patch[8 * n_t] = our_pnts[1]; // First triangle of the cell face
+        Patch[8 * n_t + 1] = our_pnts[0];
+        Patch[8 * n_t + 2] = our_pnts[2];
+
+        Patch[8 * n_t + 3] = our_pnts[2]; // Second triangle of the cell face
+        Patch[8 * n_t + 4] = our_pnts[3];
+        Patch[8 * n_t + 5] = our_pnts[1];
+        Patch[8 * n_t + 6] = -1;
+        Patch[8 * n_t + 7] = 0;
+        AllPatches[0].act_faces[0] += 1;
+
+        /* ----- TOP patch ----- */
+        AllPatches[1].p_write = AllPatches[1].p_write + 1;
+        n_t = AllPatches[1].start_idx + AllPatches[1].p_write;
+        Patch[8 * n_t + 0] = our_pnts[6];
+        Patch[8 * n_t + 1] = our_pnts[4];
+        Patch[8 * n_t + 2] = our_pnts[5];
+
+        Patch[8 * n_t + 3] = our_pnts[5];
+        Patch[8 * n_t + 4] = our_pnts[7];
+        Patch[8 * n_t + 5] = our_pnts[6];
+        Patch[8 * n_t + 6] = -2;
+        Patch[8 * n_t + 7] = 1;
+        AllPatches[1].act_faces[1] += 1;
+
+        if (i == 0) // Write a WEST EDGE patch
         {
-          break;
+          AllPatches[2].p_write = AllPatches[2].p_write + 1;
+          n_t = AllPatches[2].start_idx + AllPatches[2].p_write;
+          Patch[8 * n_t] = our_pnts[6];
+          Patch[8 * n_t + 1] = our_pnts[2];
+          Patch[8 * n_t + 2] = our_pnts[0];
+          Patch[8 * n_t + 3] = our_pnts[0];
+          Patch[8 * n_t + 4] = our_pnts[4];
+          Patch[8 * n_t + 5] = our_pnts[6];
+          Patch[8 * n_t + 6] = -3;
+          Patch[8 * n_t + 7] = 2;
+          AllPatches[2].act_faces[2] += 1;
         }
-      }
-      // If it's a bottom point keep going to find the next instance of the x-y coodrinates
-      if (km<=3) {
-        for (m=m-1; m>0; m=m-1)
+
+        if (i > 0) // Check for an WEST facing user patch
         {
-          if ((Xp_Act[3*m]==x_test)&(Xp_Act[3*m+1]==y_test))
+          test_val = *DataboxCoeff(msk, i - 1, j, 0);
+          if ((test_val == 0) | (test_val > 1))
           {
-            break;
+            if (test_val < 0)
+            {
+              return -2;
+            }                        // Error so bail
+            idx = ScanArray(test_val, patch_values, 6 + np_usr + 1);
+            if (idx < 0)
+            {
+              return -1;
+            }                   // Error so bail
+            AllPatches[idx].p_write = AllPatches[idx].p_write + 1;
+            n_t = AllPatches[idx].start_idx + AllPatches[idx].p_write;
+            Patch[8 * n_t] = our_pnts[6];
+            Patch[8 * n_t + 1] = our_pnts[2];
+            Patch[8 * n_t + 2] = our_pnts[0];
+            Patch[8 * n_t + 3] = our_pnts[0];
+            Patch[8 * n_t + 4] = our_pnts[4];
+            Patch[8 * n_t + 5] = our_pnts[6];
+            Patch[8 * n_t + 6] = test_val;
+            AllPatches[idx].act_faces[2] += 1;
+            Patch[8 * n_t + 7] = 2;
           }
         }
-      }
-
-      our_pnts[km]=m;
-
-      }
-    }
-
-    // Always add a top and bottom patch
-    /* ----- BOTTOM patch ----- */
-    AllPatches[0].p_write=AllPatches[0].p_write+1;
-    n_t=AllPatches[0].start_idx+AllPatches[0].p_write;
-    Patch[8*n_t]=our_pnts[1]; // First triangle of the cell face
-    Patch[8*n_t + 1]=our_pnts[0];
-    Patch[8*n_t + 2]=our_pnts[2];
-
-    Patch[8*n_t + 3]=our_pnts[2]; // Second triangle of the cell face
-    Patch[8*n_t + 4]=our_pnts[3];
-    Patch[8*n_t + 5]=our_pnts[1];
-    Patch[8*n_t + 6]=-1;
-    Patch[8*n_t + 7]=0;
-    AllPatches[0].act_faces[0]+=1;
-
-    /* ----- TOP patch ----- */
-    AllPatches[1].p_write=AllPatches[1].p_write+1;
-    n_t=AllPatches[1].start_idx+AllPatches[1].p_write;
-    Patch[8*n_t + 0]=our_pnts[6];
-    Patch[8*n_t + 1]=our_pnts[4];
-    Patch[8*n_t + 2]=our_pnts[5];
-
-    Patch[8*n_t + 3]=our_pnts[5];
-    Patch[8*n_t + 4]=our_pnts[7];
-    Patch[8*n_t + 5]=our_pnts[6];
-    Patch[8*n_t + 6]=-2;
-    Patch[8*n_t + 7]=1;
-    AllPatches[1].act_faces[1]+=1;
-
-    if (i==0) // Write a WEST EDGE patch
-    {
-      AllPatches[2].p_write=AllPatches[2].p_write+1;
-      n_t=AllPatches[2].start_idx+AllPatches[2].p_write;
-      Patch[8*n_t]=our_pnts[6];
-      Patch[8*n_t + 1]=our_pnts[2];
-      Patch[8*n_t + 2]=our_pnts[0];
-      Patch[8*n_t + 3]=our_pnts[0];
-      Patch[8*n_t + 4]=our_pnts[4];
-      Patch[8*n_t + 5]=our_pnts[6];
-      Patch[8*n_t + 6]=-3;
-      Patch[8*n_t + 7]=2;
-      AllPatches[2].act_faces[2]+=1;
-    }
-
-    if (i>0) // Check for an WEST facing user patch
-    {
-      test_val=*DataboxCoeff(msk, i-1, j, 0);
-      if ((test_val == 0)|(test_val > 1))
-      {
-        if (test_val<0) {return -2;} // Error so bail
-        idx=ScanArray(test_val,patch_values,6+np_usr+1);
-        if (idx<0) {return -1;} // Error so bail
-        AllPatches[idx].p_write=AllPatches[idx].p_write+1;
-        n_t=AllPatches[idx].start_idx+AllPatches[idx].p_write;
-        Patch[8*n_t]=our_pnts[6];
-        Patch[8*n_t + 1]=our_pnts[2];
-        Patch[8*n_t + 2]=our_pnts[0];
-        Patch[8*n_t + 3]=our_pnts[0];
-        Patch[8*n_t + 4]=our_pnts[4];
-        Patch[8*n_t + 5]=our_pnts[6];
-        Patch[8*n_t + 6]=test_val;
-        AllPatches[idx].act_faces[2]+=1;
-        Patch[8*n_t + 7]=2;
-      }
-    }
-
-    if (i==(NX-1)) // Write an EAST EDGE patch
-    {
-      AllPatches[3].p_write=AllPatches[3].p_write+1;
-      n_t=AllPatches[3].start_idx+AllPatches[3].p_write;
-      Patch[8*n_t]=our_pnts[1];
-      Patch[8*n_t + 1]=our_pnts[3];
-      Patch[8*n_t + 2]=our_pnts[7];
-      Patch[8*n_t + 3]=our_pnts[7];
-      Patch[8*n_t + 4]=our_pnts[5];
-      Patch[8*n_t + 5]=our_pnts[1];
-      Patch[8*n_t + 6]=-4;
-      Patch[8*n_t + 7]=3;
-      AllPatches[3].act_faces[3]+=1;
-    }
 
-    if (i<(NX-1)) // Check for an EAST facing user patch
-    {
-      test_val=*DataboxCoeff(msk, i+1, j, 0);
-      if ((test_val == 0)|(test_val > 1))
-      {
-        if (test_val<0) {return -2;} // Error so bail
-        idx=ScanArray(test_val,patch_values,6+np_usr+1);
-        if (idx<0) {return -1;} // Error so bail
-        AllPatches[idx].p_write=AllPatches[idx].p_write+1;
-        n_t=AllPatches[idx].start_idx+AllPatches[idx].p_write;
-        Patch[8*n_t]=our_pnts[1];
-        Patch[8*n_t + 1]=our_pnts[3];
-        Patch[8*n_t + 2]=our_pnts[7];
-        Patch[8*n_t + 3]=our_pnts[7];
-        Patch[8*n_t + 4]=our_pnts[5];
-        Patch[8*n_t + 5]=our_pnts[1];
-        Patch[8*n_t + 6]=test_val;
-        Patch[8*n_t + 7]=3;
-        AllPatches[idx].act_faces[3]+=1;
-      }
-    }
+        if (i == (NX - 1)) // Write an EAST EDGE patch
+        {
+          AllPatches[3].p_write = AllPatches[3].p_write + 1;
+          n_t = AllPatches[3].start_idx + AllPatches[3].p_write;
+          Patch[8 * n_t] = our_pnts[1];
+          Patch[8 * n_t + 1] = our_pnts[3];
+          Patch[8 * n_t + 2] = our_pnts[7];
+          Patch[8 * n_t + 3] = our_pnts[7];
+          Patch[8 * n_t + 4] = our_pnts[5];
+          Patch[8 * n_t + 5] = our_pnts[1];
+          Patch[8 * n_t + 6] = -4;
+          Patch[8 * n_t + 7] = 3;
+          AllPatches[3].act_faces[3] += 1;
+        }
 
-    if (j==0) // Write a SOUTH EDGE patch
-    {
-      AllPatches[4].p_write=AllPatches[4].p_write+1;
-      n_t=AllPatches[4].start_idx+AllPatches[4].p_write;
-      Patch[8*n_t]=our_pnts[4];
-      Patch[8*n_t + 1]=our_pnts[0];
-      Patch[8*n_t + 2]=our_pnts[1];
-      Patch[8*n_t + 3]=our_pnts[1];
-      Patch[8*n_t + 4]=our_pnts[5];
-      Patch[8*n_t + 5]=our_pnts[4];
-      Patch[8*n_t + 6]=-5;
-      Patch[8*n_t + 7]=4;
-      AllPatches[4].act_faces[4]+=1;
-    }
+        if (i < (NX - 1)) // Check for an EAST facing user patch
+        {
+          test_val = *DataboxCoeff(msk, i + 1, j, 0);
+          if ((test_val == 0) | (test_val > 1))
+          {
+            if (test_val < 0)
+            {
+              return -2;
+            }                        // Error so bail
+            idx = ScanArray(test_val, patch_values, 6 + np_usr + 1);
+            if (idx < 0)
+            {
+              return -1;
+            }                   // Error so bail
+            AllPatches[idx].p_write = AllPatches[idx].p_write + 1;
+            n_t = AllPatches[idx].start_idx + AllPatches[idx].p_write;
+            Patch[8 * n_t] = our_pnts[1];
+            Patch[8 * n_t + 1] = our_pnts[3];
+            Patch[8 * n_t + 2] = our_pnts[7];
+            Patch[8 * n_t + 3] = our_pnts[7];
+            Patch[8 * n_t + 4] = our_pnts[5];
+            Patch[8 * n_t + 5] = our_pnts[1];
+            Patch[8 * n_t + 6] = test_val;
+            Patch[8 * n_t + 7] = 3;
+            AllPatches[idx].act_faces[3] += 1;
+          }
+        }
 
-    if (j>0) // Check for an SOUTH facing user patch
-    {
-      test_val=*DataboxCoeff(msk, i, j-1, 0);
-      if ((test_val == 0)|(test_val > 1))
-      {
-        if (test_val<0) {return -2;} // Error so bail
-        idx=ScanArray(test_val,patch_values,6+np_usr+1);
-        if (idx<0) {return -1;} // Error so bail
-        AllPatches[idx].p_write=AllPatches[idx].p_write+1;
-        n_t=AllPatches[idx].start_idx+AllPatches[idx].p_write;
-        Patch[8*n_t]=our_pnts[4];
-        Patch[8*n_t + 1]=our_pnts[0];
-        Patch[8*n_t + 2]=our_pnts[1];
-        Patch[8*n_t + 3]=our_pnts[1];
-        Patch[8*n_t + 4]=our_pnts[5];
-        Patch[8*n_t + 5]=our_pnts[4];
-        Patch[8*n_t + 6]=test_val;
-        Patch[8*n_t + 7]=4;
-        AllPatches[idx].act_faces[4]+=1;
-      }
-    }
+        if (j == 0) // Write a SOUTH EDGE patch
+        {
+          AllPatches[4].p_write = AllPatches[4].p_write + 1;
+          n_t = AllPatches[4].start_idx + AllPatches[4].p_write;
+          Patch[8 * n_t] = our_pnts[4];
+          Patch[8 * n_t + 1] = our_pnts[0];
+          Patch[8 * n_t + 2] = our_pnts[1];
+          Patch[8 * n_t + 3] = our_pnts[1];
+          Patch[8 * n_t + 4] = our_pnts[5];
+          Patch[8 * n_t + 5] = our_pnts[4];
+          Patch[8 * n_t + 6] = -5;
+          Patch[8 * n_t + 7] = 4;
+          AllPatches[4].act_faces[4] += 1;
+        }
 
-    if (j==(NY-1)) // Write a NORTH EDGE patch
-    {
-      AllPatches[5].p_write=AllPatches[5].p_write+1;
-      n_t=AllPatches[5].start_idx+AllPatches[5].p_write;
-      Patch[8*n_t]=our_pnts[3];
-      Patch[8*n_t + 1]=our_pnts[2];
-      Patch[8*n_t + 2]=our_pnts[6];
-
-      Patch[8*n_t + 3]=our_pnts[6];
-      Patch[8*n_t + 4]=our_pnts[7];
-      Patch[8*n_t + 5]=our_pnts[3];
-      Patch[8*n_t + 6]=-6;
-      Patch[8*n_t + 7]=5;
-      AllPatches[5].act_faces[5]+=1;
-    }
+        if (j > 0) // Check for an SOUTH facing user patch
+        {
+          test_val = *DataboxCoeff(msk, i, j - 1, 0);
+          if ((test_val == 0) | (test_val > 1))
+          {
+            if (test_val < 0)
+            {
+              return -2;
+            }                        // Error so bail
+            idx = ScanArray(test_val, patch_values, 6 + np_usr + 1);
+            if (idx < 0)
+            {
+              return -1;
+            }                   // Error so bail
+            AllPatches[idx].p_write = AllPatches[idx].p_write + 1;
+            n_t = AllPatches[idx].start_idx + AllPatches[idx].p_write;
+            Patch[8 * n_t] = our_pnts[4];
+            Patch[8 * n_t + 1] = our_pnts[0];
+            Patch[8 * n_t + 2] = our_pnts[1];
+            Patch[8 * n_t + 3] = our_pnts[1];
+            Patch[8 * n_t + 4] = our_pnts[5];
+            Patch[8 * n_t + 5] = our_pnts[4];
+            Patch[8 * n_t + 6] = test_val;
+            Patch[8 * n_t + 7] = 4;
+            AllPatches[idx].act_faces[4] += 1;
+          }
+        }
 
-    if (j<(NY-1)) // Check for an NORTH facing user patch
-    {
-      test_val=*DataboxCoeff(msk, i, j+1, 0);
-      if ((test_val == 0)|(test_val > 1))
-      {
-        if (test_val<0) {return -2;} // Error so bail
-        idx=ScanArray(test_val,patch_values,6+np_usr+1);
-        if (idx<0) {return -1;} // Error so bail
-        AllPatches[idx].p_write=AllPatches[idx].p_write+1;
-        n_t=AllPatches[idx].start_idx+AllPatches[idx].p_write;
-        Patch[8*n_t]=our_pnts[3];
-        Patch[8*n_t + 1]=our_pnts[2];
-        Patch[8*n_t + 2]=our_pnts[6];
-        Patch[8*n_t + 3]=our_pnts[6];
-        Patch[8*n_t + 4]=our_pnts[7];
-        Patch[8*n_t + 5]=our_pnts[3];
-        Patch[8*n_t + 6]=test_val;
-        AllPatches[idx].act_faces[5]+=1;
-        Patch[8*n_t + 7]=5;
-      }
-    }
+        if (j == (NY - 1)) // Write a NORTH EDGE patch
+        {
+          AllPatches[5].p_write = AllPatches[5].p_write + 1;
+          n_t = AllPatches[5].start_idx + AllPatches[5].p_write;
+          Patch[8 * n_t] = our_pnts[3];
+          Patch[8 * n_t + 1] = our_pnts[2];
+          Patch[8 * n_t + 2] = our_pnts[6];
+
+          Patch[8 * n_t + 3] = our_pnts[6];
+          Patch[8 * n_t + 4] = our_pnts[7];
+          Patch[8 * n_t + 5] = our_pnts[3];
+          Patch[8 * n_t + 6] = -6;
+          Patch[8 * n_t + 7] = 5;
+          AllPatches[5].act_faces[5] += 1;
+        }
 
-    test_val=0;
+        if (j < (NY - 1)) // Check for an NORTH facing user patch
+        {
+          test_val = *DataboxCoeff(msk, i, j + 1, 0);
+          if ((test_val == 0) | (test_val > 1))
+          {
+            if (test_val < 0)
+            {
+              return -2;
+            }                        // Error so bail
+            idx = ScanArray(test_val, patch_values, 6 + np_usr + 1);
+            if (idx < 0)
+            {
+              return -1;
+            }                   // Error so bail
+            AllPatches[idx].p_write = AllPatches[idx].p_write + 1;
+            n_t = AllPatches[idx].start_idx + AllPatches[idx].p_write;
+            Patch[8 * n_t] = our_pnts[3];
+            Patch[8 * n_t + 1] = our_pnts[2];
+            Patch[8 * n_t + 2] = our_pnts[6];
+            Patch[8 * n_t + 3] = our_pnts[6];
+            Patch[8 * n_t + 4] = our_pnts[7];
+            Patch[8 * n_t + 5] = our_pnts[3];
+            Patch[8 * n_t + 6] = test_val;
+            AllPatches[idx].act_faces[5] += 1;
+            Patch[8 * n_t + 7] = 5;
+          }
+        }
 
-    } // End of mask_val==1 block
-  } // End of i loop
+        test_val = 0;
+      } // End of mask_val==1 block
+    } // End of i loop
   } // End of j loop
 
   // if (debugger==1) {
@@ -737,384 +863,464 @@ int            MakePatchySolid(
   //   printf("Elapsed time: %f\n", run_time);
   // }
 
-  if (debugger==1) printf("DBG: Done building point-patch database\n");
+  if (debugger == 1)
+    printf("DBG: Done building point-patch database\n");
 
-  char *dir_labels[]={"Bottom","Top","Left","Right","Front","Back"};
+  char *dir_labels[] = { "Bottom", "Top", "Left", "Right", "Front", "Back" };
 
   // --------------------------------------------------------------------------
   //              ===== Write out the solid file =====
   // --------------------------------------------------------------------------
   // Write the patch order to the terminal so the user can copy them
   printf("PFPATCHYSOLID - Patch write order is:\n\n");
-  if (AllPatches[0].patch_cell_count>0) {printf(" Bottom ");}
-  if (AllPatches[1].patch_cell_count>0) {printf(" Top ");}
+  if (AllPatches[0].patch_cell_count > 0)
+  {
+    printf(" Bottom ");
+  }
+  if (AllPatches[1].patch_cell_count > 0)
+  {
+    printf(" Top ");
+  }
 
-  if (AllPatches[2].patch_cell_count>0) {printf(" Left ");}  // West
-  if (AllPatches[3].patch_cell_count>0) {printf(" Right ");} // East
+  if (AllPatches[2].patch_cell_count > 0)
+  {
+    printf(" Left ");
+  }                                                          // West
+  if (AllPatches[3].patch_cell_count > 0)
+  {
+    printf(" Right ");
+  }                                                          // East
 
-  if (AllPatches[4].patch_cell_count>0) {printf(" Front ");} // South
-  if (AllPatches[5].patch_cell_count>0) {printf(" Back ");}  // North
-  for (i=6; i<(6+np_usr+1); ++i)
+  if (AllPatches[4].patch_cell_count > 0)
+  {
+    printf(" Front ");
+  }                                                          // South
+  if (AllPatches[5].patch_cell_count > 0)
   {
-    if (sub_patches==1) {
-      if ((AllPatches[i].patch_cell_count>0)&(AllPatches[i].value!=0))
+    printf(" Back ");
+  }                                                          // North
+  for (i = 6; i < (6 + np_usr + 1); ++i)
+  {
+    if (sub_patches == 1)
+    {
+      if ((AllPatches[i].patch_cell_count > 0) & (AllPatches[i].value != 0))
       {
-        for (j=0; j<6; ++j) {
-        if (AllPatches[i].act_faces[j]>0)
-        printf(" Usr_%i_%s ",AllPatches[i].value,dir_labels[j]);
+        for (j = 0; j < 6; ++j)
+        {
+          if (AllPatches[i].act_faces[j] > 0)
+            printf(" Usr_%i_%s ", AllPatches[i].value, dir_labels[j]);
         }
       }
-    } else {
-    if ((AllPatches[i].patch_cell_count>0)&(AllPatches[i].value!=0))
-    {printf(" Usr_%i ",AllPatches[i].value);}
+    }
+    else
+    {
+      if ((AllPatches[i].patch_cell_count > 0) & (AllPatches[i].value != 0))
+      {
+        printf(" Usr_%i ", AllPatches[i].value);
+      }
     }
   }
   printf("\n \n");
 
   // ====== Now write out the solid =====
-  if (bin_out==0)
-  {
-  fprintf(fp,"1 \n"); // VERSION #
-  fprintf(fp,"%i \n",np_act+1);   // # of POINTS/VERTICIES
-  for (i=0; i<=np_act; ++i)
+  if (bin_out == 0)
   {
-    fprintf(fp,"%17.4f %17.4f %17.4f\n",Xp_Act[3*i],Xp_Act[3*i+1],Xp_Act[3*i+2]);
-  }
-  fprintf(fp,"1 \n"); // Number of SOLIDS (only one allowed here)
-  fprintf(fp,"%i \n",(cell_faces)*2); // Total number of triangles
+    fprintf(fp, "1 \n"); // VERSION #
+    fprintf(fp, "%i \n", np_act + 1); // # of POINTS/VERTICIES
+    for (i = 0; i <= np_act; ++i)
+    {
+      fprintf(fp, "%17.4f %17.4f %17.4f\n", Xp_Act[3 * i], Xp_Act[3 * i + 1], Xp_Act[3 * i + 2]);
+    }
+    fprintf(fp, "1 \n"); // Number of SOLIDS (only one allowed here)
+    fprintf(fp, "%i \n", (cell_faces) * 2); // Total number of triangles
 
-  for (i=0; i<cell_faces; ++i)
-  {
-  fprintf(fp," %i %i %i\n",Patch[8*i],Patch[8*i + 1],Patch[8*i + 2]);
-  fprintf(fp," %i %i %i\n",Patch[8*i + 3],Patch[8*i + 4],Patch[8*i + 5]);
-  }
+    for (i = 0; i < cell_faces; ++i)
+    {
+      fprintf(fp, " %i %i %i\n", Patch[8 * i], Patch[8 * i + 1], Patch[8 * i + 2]);
+      fprintf(fp, " %i %i %i\n", Patch[8 * i + 3], Patch[8 * i + 4], Patch[8 * i + 5]);
+    }
 
-  // NEED TO MODIFY FOR SUB PATCH OPTION
-  if (sub_patches==1) {
-    int all_faces=0;
-    // Count up all the ACTUAL patches
-    for (i=0; i<(6+np_usr+1); ++i) {
-      // printf("F%i, ",i);
-      for (j=0; j<6; ++j) {
-        // printf(" %i ",AllPatches[i].act_faces[j]);
-        if ( (AllPatches[i].value!=0) && (AllPatches[i].act_faces[j]>0)) {all_faces+=1;}
+    // NEED TO MODIFY FOR SUB PATCH OPTION
+    if (sub_patches == 1)
+    {
+      int all_faces = 0;
+      // Count up all the ACTUAL patches
+      for (i = 0; i < (6 + np_usr + 1); ++i)
+      {
+        // printf("F%i, ",i);
+        for (j = 0; j < 6; ++j)
+        {
+          // printf(" %i ",AllPatches[i].act_faces[j]);
+          if ((AllPatches[i].value != 0) && (AllPatches[i].act_faces[j] > 0))
+          {
+            all_faces += 1;
+          }
+        }
+        // printf("    Val=%i, Cnt=%i \n",AllPatches[i].value,AllPatches[i].patch_cell_count);
       }
-      // printf("    Val=%i, Cnt=%i \n",AllPatches[i].value,AllPatches[i].patch_cell_count);
-    }
 
-    // printf("non_blanks=%i, all_faces=%i \n",non_blanks,all_faces);
+      // printf("non_blanks=%i, all_faces=%i \n",non_blanks,all_faces);
 
-    fprintf(fp,"%i \n",all_faces); // -1 since zero is omitted
+      fprintf(fp, "%i \n", all_faces); // -1 since zero is omitted
 
-    for (i=0; i<(6+np_usr+1); ++i)
-    {
-      if ((AllPatches[i].patch_cell_count>0)&(AllPatches[i].value!=0))
+      for (i = 0; i < (6 + np_usr + 1); ++i)
       {
-        for (k=0; k<6; ++k) {
-          if (AllPatches[i].act_faces[k]>0) {
-            fprintf(fp,"%i \n",AllPatches[i].act_faces[k]*2);
-            for (j=AllPatches[i].start_idx; j<=AllPatches[i].end_idx; ++j)
+        if ((AllPatches[i].patch_cell_count > 0) & (AllPatches[i].value != 0))
+        {
+          for (k = 0; k < 6; ++k)
+          {
+            if (AllPatches[i].act_faces[k] > 0)
             {
-              if (Patch[8*j + 7]==k) {
-              fprintf(fp,"%i \n",2*j);
-              fprintf(fp,"%i \n",2*j+1);
+              fprintf(fp, "%i \n", AllPatches[i].act_faces[k] * 2);
+              for (j = AllPatches[i].start_idx; j <= AllPatches[i].end_idx; ++j)
+              {
+                if (Patch[8 * j + 7] == k)
+                {
+                  fprintf(fp, "%i \n", 2 * j);
+                  fprintf(fp, "%i \n", 2 * j + 1);
+                }
               }
             }
-          }
-        } // End of k loop
+          } // End of k loop
+        }
       }
+      // ----- End of sub patch section for ASCII write -----
     }
-  // ----- End of sub patch section for ASCII write -----
-  } else {
-  if (any_zeros==1)
-  {
-    fprintf(fp,"%i \n",non_blanks-1); // -1 since zero is omitted
-  } else {
-    fprintf(fp,"%i \n",non_blanks); // -1 since zero is omitted
-  }
-
-  for (i=0; i<(6+np_usr+1); ++i)
-  {
-    if ((AllPatches[i].patch_cell_count>0)&(AllPatches[i].value!=0))
+    else
     {
-      fprintf(fp,"%i \n",AllPatches[i].patch_cell_count*2);
-      for (j=2*AllPatches[i].start_idx; j<=2*AllPatches[i].end_idx+1; ++j)
+      if (any_zeros == 1)
+      {
+        fprintf(fp, "%i \n", non_blanks - 1); // -1 since zero is omitted
+      }
+      else
       {
-        fprintf(fp,"%i \n",j);
+        fprintf(fp, "%i \n", non_blanks); // -1 since zero is omitted
+      }
+
+      for (i = 0; i < (6 + np_usr + 1); ++i)
+      {
+        if ((AllPatches[i].patch_cell_count > 0) & (AllPatches[i].value != 0))
+        {
+          fprintf(fp, "%i \n", AllPatches[i].patch_cell_count * 2);
+          for (j = 2 * AllPatches[i].start_idx; j <= 2 * AllPatches[i].end_idx + 1; ++j)
+          {
+            fprintf(fp, "%i \n", j);
+          }
+        }
       }
     }
+    if (debugger == 1)
+      printf("DBG: Done writing ASCII solid\n");
   }
-  }
-  if (debugger==1) printf("DBG: Done writing ASCII solid\n");
-  } else {
+  else
+  {
     /* ---------- PLACE HOLDER for future feature ---------- */
     // This writes a BINARY solid file. The file can be written and read with
     // ParFlow BUT a bug limits this to a processor topology of (1,1,1)
     //  * * A fix is coming so just use ASCII solid files for now * *
 
-    int write_int=1;
+    int write_int = 1;
 
-    tools_WriteInt(fp,&write_int, 1); // VERSION #
-    write_int=np_act+1;
-    tools_WriteInt(fp,&write_int, 1); // # of POINTS/VERTICIES
-    for (i=0; i<=np_act; ++i)
+    tools_WriteInt(fp, &write_int, 1); // VERSION #
+    write_int = np_act + 1;
+    tools_WriteInt(fp, &write_int, 1); // # of POINTS/VERTICIES
+    for (i = 0; i <= np_act; ++i)
     {
-      tools_WriteDouble(fp,&Xp_Act[3*i], 1);
-      tools_WriteDouble(fp,&Xp_Act[3*i+1], 1);
-      tools_WriteDouble(fp,&Xp_Act[3*i+2], 1);
+      tools_WriteDouble(fp, &Xp_Act[3 * i], 1);
+      tools_WriteDouble(fp, &Xp_Act[3 * i + 1], 1);
+      tools_WriteDouble(fp, &Xp_Act[3 * i + 2], 1);
     }
-    write_int=1;
-    tools_WriteInt(fp,&write_int, 1); // Number of SOLIDS (only one allowed here)
-    write_int=cell_faces*2;
-    tools_WriteInt(fp,&write_int, 1); // Total number of triangles
-    for (i=0; i<(cell_faces); ++i)
+    write_int = 1;
+    tools_WriteInt(fp, &write_int, 1); // Number of SOLIDS (only one allowed here)
+    write_int = cell_faces * 2;
+    tools_WriteInt(fp, &write_int, 1); // Total number of triangles
+    for (i = 0; i < (cell_faces); ++i)
     {
-      tools_WriteInt(fp,&Patch[8*i], 1);
-      tools_WriteInt(fp,&Patch[8*i + 1], 1);
-      tools_WriteInt(fp,&Patch[8*i + 2], 1);
-      tools_WriteInt(fp,&Patch[8*i + 3], 1);
-      tools_WriteInt(fp,&Patch[8*i + 4], 1);
-      tools_WriteInt(fp,&Patch[8*i + 5], 1);
+      tools_WriteInt(fp, &Patch[8 * i], 1);
+      tools_WriteInt(fp, &Patch[8 * i + 1], 1);
+      tools_WriteInt(fp, &Patch[8 * i + 2], 1);
+      tools_WriteInt(fp, &Patch[8 * i + 3], 1);
+      tools_WriteInt(fp, &Patch[8 * i + 4], 1);
+      tools_WriteInt(fp, &Patch[8 * i + 5], 1);
     }
 
-    if (sub_patches==1) {
-        int all_faces=0;
-        // Count up all the ACTUAL patches
-        for (i=0; i<(6+np_usr+1); ++i) {
-          for (j=0; j<6; ++j) {
-            if ((AllPatches[i].value!=0) && (AllPatches[i].act_faces[j]>0)) {all_faces+=1;}
+    if (sub_patches == 1)
+    {
+      int all_faces = 0;
+      // Count up all the ACTUAL patches
+      for (i = 0; i < (6 + np_usr + 1); ++i)
+      {
+        for (j = 0; j < 6; ++j)
+        {
+          if ((AllPatches[i].value != 0) && (AllPatches[i].act_faces[j] > 0))
+          {
+            all_faces += 1;
           }
         }
+      }
 
-        write_int=all_faces;
-        tools_WriteInt(fp,&write_int, 1); // -1 since zero is omitted
+      write_int = all_faces;
+      tools_WriteInt(fp, &write_int, 1);  // -1 since zero is omitted
 
-        for (i=0; i<(6+np_usr+1); ++i)
+      for (i = 0; i < (6 + np_usr + 1); ++i)
+      {
+        if ((AllPatches[i].patch_cell_count > 0) & (AllPatches[i].value != 0))
         {
-          if ((AllPatches[i].patch_cell_count>0)&(AllPatches[i].value!=0))
+          for (k = 0; k < 6; ++k)
           {
-            for (k=0; k<6; ++k) {
-              if (AllPatches[i].act_faces[k]>0) {
-                write_int=AllPatches[i].act_faces[k]*2;
-                tools_WriteInt(fp,&write_int, 1);
-                for (j=AllPatches[i].start_idx; j<=AllPatches[i].end_idx; ++j)
+            if (AllPatches[i].act_faces[k] > 0)
+            {
+              write_int = AllPatches[i].act_faces[k] * 2;
+              tools_WriteInt(fp, &write_int, 1);
+              for (j = AllPatches[i].start_idx; j <= AllPatches[i].end_idx; ++j)
+              {
+                if (Patch[8 * j + 7] == k)
                 {
-                  if (Patch[8*j + 7]==k) {
-                  write_int=2*j;
-                  tools_WriteInt(fp,&write_int, 1);
-                  write_int=2*j+1;
-                  tools_WriteInt(fp,&write_int, 1);
-                  }
+                  write_int = 2 * j;
+                  tools_WriteInt(fp, &write_int, 1);
+                  write_int = 2 * j + 1;
+                  tools_WriteInt(fp, &write_int, 1);
                 }
               }
-            } // End of k loop
-          }
+            }
+          }   // End of k loop
         }
+      }
       // ----- End of sub patch section for BINARY write -----
-
-    } else {
-    if (any_zeros==1)
-    {
-      write_int=non_blanks-1;
-    } else {
-      write_int=non_blanks;
     }
-    tools_WriteInt(fp,&write_int, 1); // -1 since zero is omitted
-
-    for (i=0; i<(6+np_usr+1); ++i)
+    else
     {
-      if ((AllPatches[i].patch_cell_count>0)&(AllPatches[i].value!=0))
+      if (any_zeros == 1)
+      {
+        write_int = non_blanks - 1;
+      }
+      else
       {
-        write_int=AllPatches[i].patch_cell_count*2;
-        tools_WriteInt(fp,&write_int, 1);
-        for (j=2*AllPatches[i].start_idx; j<=2*AllPatches[i].end_idx+1; ++j)
+        write_int = non_blanks;
+      }
+      tools_WriteInt(fp, &write_int, 1); // -1 since zero is omitted
+
+      for (i = 0; i < (6 + np_usr + 1); ++i)
+      {
+        if ((AllPatches[i].patch_cell_count > 0) & (AllPatches[i].value != 0))
         {
-          write_int=j;
-          tools_WriteInt(fp,&write_int, 1);
+          write_int = AllPatches[i].patch_cell_count * 2;
+          tools_WriteInt(fp, &write_int, 1);
+          for (j = 2 * AllPatches[i].start_idx; j <= 2 * AllPatches[i].end_idx + 1; ++j)
+          {
+            write_int = j;
+            tools_WriteInt(fp, &write_int, 1);
+          }
         }
       }
-    }
-    if (debugger==1) printf("DBG: Done writing BINARY solid\n");
-  }  // End of ascii/binary solid file test
+      if (debugger == 1)
+        printf("DBG: Done writing BINARY solid\n");
+    } // End of ascii/binary solid file test
   }
   // --------------------------------------------------------------------------
   //        ===== Write the VTK if the file ID is not NULL =====
   // --------------------------------------------------------------------------
-  if (fp_vtk!=NULL) {
-
-  // If an ASCII is preferred you can easily change that here
-  int write_ascii=0;      // Default is 0 to write a binary
-  int write_float=1;      // Only compatible with BINARY, but writes data as float to save space
-
-  if (debugger==1) {
-    if (write_ascii==1) {
-       printf("DBG: Writing ASCII vtk\n");
-     } else {
-       printf("DBG: Writing BINARY vtk \n");
-       if (write_float==1) {
-         printf("DBG: -> Storing vtk points as FLOAT \n");
-       } else {
-         printf("DBG: -> Storing vtk points as DOUBLE \n");
-       }
-     }
-   }
-
-  double *PatchEl;   // patch mean elevations
-  int *PatchVal;    // patch ID numbers
-  PatchEl=(double*)calloc(cell_faces*2,sizeof(double));
-  PatchVal=(int*)calloc(cell_faces*2,sizeof(int));
-
-  k=0;
-  for (i=0; i<cell_faces*2; i=i+2)
-  {
-    PatchEl[i]=(Xp_Act[3*Patch[8*k]+2]+Xp_Act[3*Patch[8*k+1]+2]+Xp_Act[3*Patch[8*k+2]+2])/3.0;
-    PatchEl[i+1]=(Xp_Act[3*Patch[8*k+3]+2]+Xp_Act[3*Patch[8*k+4]+2]+Xp_Act[3*Patch[8*k+5]+2])/3.0;
-
-    PatchVal[i]=Patch[8*k + 6];
-    PatchVal[i+1]=Patch[8*k + 6];
-    k=k+1;
-  }
-
-  float *fPatchEl;
-  int *fPatchVal;
-  float *fXp_Act;
-
-  if (write_float==1)
+  if (fp_vtk != NULL)
   {
-    fPatchEl=(float*)calloc(cell_faces*2,sizeof(float));
-    fXp_Act=(float*)calloc((np_act+1)*3,sizeof(float));
-    fPatchVal=(int*)calloc(cell_faces*2,sizeof(int));
+    // If an ASCII is preferred you can easily change that here
+    int write_ascii = 0;  // Default is 0 to write a binary
+    int write_float = 1;  // Only compatible with BINARY, but writes data as float to save space
 
-    for (i=0; i<=(np_act); i++)
+    if (debugger == 1)
     {
-      fXp_Act[3*i]=(float)Xp_Act[3*i];
-      fXp_Act[3*i+1]=(float)Xp_Act[3*i+1];
-      fXp_Act[3*i+2]=(float)Xp_Act[3*i+2];
+      if (write_ascii == 1)
+      {
+        printf("DBG: Writing ASCII vtk\n");
+      }
+      else
+      {
+        printf("DBG: Writing BINARY vtk \n");
+        if (write_float == 1)
+        {
+          printf("DBG: -> Storing vtk points as FLOAT \n");
+        }
+        else
+        {
+          printf("DBG: -> Storing vtk points as DOUBLE \n");
+        }
+      }
     }
 
-    for (i=0; i<cell_faces*2; ++i)
+    double *PatchEl; // patch mean elevations
+    int *PatchVal;  // patch ID numbers
+    PatchEl = (double*)calloc(cell_faces * 2, sizeof(double));
+    PatchVal = (int*)calloc(cell_faces * 2, sizeof(int));
+
+    k = 0;
+    for (i = 0; i < cell_faces * 2; i = i + 2)
     {
-      fPatchEl[i]=(float)PatchEl[i];
-      fPatchVal[i]=(int)PatchVal[i];
+      PatchEl[i] = (Xp_Act[3 * Patch[8 * k] + 2] + Xp_Act[3 * Patch[8 * k + 1] + 2] + Xp_Act[3 * Patch[8 * k + 2] + 2]) / 3.0;
+      PatchEl[i + 1] = (Xp_Act[3 * Patch[8 * k + 3] + 2] + Xp_Act[3 * Patch[8 * k + 4] + 2] + Xp_Act[3 * Patch[8 * k + 5] + 2]) / 3.0;
+
+      PatchVal[i] = Patch[8 * k + 6];
+      PatchVal[i + 1] = Patch[8 * k + 6];
+      k = k + 1;
     }
-  }
 
-  if (debugger==1) printf("DBG: Done with VTK prep, writing file...\n");
+    float *fPatchEl;
+    int *fPatchVal;
+    float *fXp_Act;
 
-  //This uses the mixed VTK BINARY legacy format, writes as either double or float
-  fprintf(fp_vtk, "# vtk DataFile Version 2.0\n");
-  fprintf(fp_vtk, "ParFlow Solid file VTK output\n");
-  if (write_ascii==1) {fprintf(fp_vtk, "ASCII\n");} else {fprintf(fp_vtk, "BINARY\n");}
-  fprintf(fp_vtk, "DATASET POLYDATA\n");
-  if (write_ascii==1) { // ASCII VTK
-  fprintf(fp_vtk,"%s %i %s\n","POINTS",np_act+1,"float");
-    for (i=0; i<=np_act; ++i)
-    {
-      fprintf(fp_vtk,"%17.4f %17.4f %17.4f\n",Xp_Act[3*i],Xp_Act[3*i+1],Xp_Act[3*i+2]);
-    }
-  } else {
-  // BINARY VTK
-  if (write_float) {
-    fprintf(fp_vtk,"%s %i %s\n","POINTS",np_act+1,"float");
-    for (i=0; i<=np_act; ++i)
+    if (write_float == 1)
     {
-      for (j=0; j<3; ++j)
+      fPatchEl = (float*)calloc(cell_faces * 2, sizeof(float));
+      fXp_Act = (float*)calloc((np_act + 1) * 3, sizeof(float));
+      fPatchVal = (int*)calloc(cell_faces * 2, sizeof(int));
+
+      for (i = 0; i <= (np_act); i++)
       {
-        tools_WriteFloat(fp_vtk,&fXp_Act[3*i+j], 1);
+        fXp_Act[3 * i] = (float)Xp_Act[3 * i];
+        fXp_Act[3 * i + 1] = (float)Xp_Act[3 * i + 1];
+        fXp_Act[3 * i + 2] = (float)Xp_Act[3 * i + 2];
       }
-    }
-  } else {
-    fprintf(fp_vtk,"%s %i %s\n","POINTS",np_act+1,"double");
-    for (i=0; i<=np_act; ++i)
-    {
-      for (j=0; j<3; ++j)
+
+      for (i = 0; i < cell_faces * 2; ++i)
       {
-        tools_WriteDouble(fp_vtk,&Xp_Act[3*i+j], 1);
+        fPatchEl[i] = (float)PatchEl[i];
+        fPatchVal[i] = (int)PatchVal[i];
       }
     }
-  }
-  }
 
-  int vrtx=3;
-  int* nvrtx=&vrtx;
-  fprintf(fp_vtk,"%s %i %i\n","POLYGONS",(cell_faces)*2,(cell_faces)*2*4);
-  if (write_ascii==1) { // ASCII VTK
-    for (i=0; i<cell_faces; ++i)
+    if (debugger == 1)
+      printf("DBG: Done with VTK prep, writing file...\n");
+
+    //This uses the mixed VTK BINARY legacy format, writes as either double or float
+    fprintf(fp_vtk, "# vtk DataFile Version 2.0\n");
+    fprintf(fp_vtk, "ParFlow Solid file VTK output\n");
+    if (write_ascii == 1)
     {
-    fprintf(fp_vtk," %i %i %i %i\n",*nvrtx,Patch[8*i],Patch[8*i+1],Patch[8*i+2]);
-    fprintf(fp_vtk," %i %i %i %i\n",*nvrtx,Patch[8*i+3],Patch[8*i+4],Patch[8*i+5]);
+      fprintf(fp_vtk, "ASCII\n");
     }
-  } else {
-  // BINARY VTK
-  for (i=0; i<cell_faces; ++i)
-  {
-    tools_WriteInt(fp_vtk,nvrtx, 1);
-    for (j=0; j<3; ++j)
+    else
     {
-      tools_WriteInt(fp_vtk,&Patch[8*i+j], 1);
+      fprintf(fp_vtk, "BINARY\n");
     }
-    tools_WriteInt(fp_vtk,nvrtx, 1);
-    for (j=3; j<6; ++j)
+    fprintf(fp_vtk, "DATASET POLYDATA\n");
+    if (write_ascii == 1) // ASCII VTK
     {
-      tools_WriteInt(fp_vtk,&Patch[8*i+j], 1);
+      fprintf(fp_vtk, "%s %i %s\n", "POINTS", np_act + 1, "float");
+      for (i = 0; i <= np_act; ++i)
+      {
+        fprintf(fp_vtk, "%17.4f %17.4f %17.4f\n", Xp_Act[3 * i], Xp_Act[3 * i + 1], Xp_Act[3 * i + 2]);
+      }
     }
-  }
-  }
-
-  // Now write the cell data
-  fprintf(fp_vtk,"%s %i\n","CELL_DATA ",(cell_faces)*2);
-  if (write_ascii==1) { // ASCII VTK
-    fprintf(fp_vtk,"%s %s %s\n","SCALARS","Elev","float");
-    fprintf(fp_vtk,"LOOKUP_TABLE default\n");
-    for (i=0; i<cell_faces*2; ++i)
+    else
     {
-      fprintf(fp_vtk,"%17.4f \n",PatchEl[i]);
+      // BINARY VTK
+      if (write_float)
+      {
+        fprintf(fp_vtk, "%s %i %s\n", "POINTS", np_act + 1, "float");
+        for (i = 0; i <= np_act; ++i)
+        {
+          for (j = 0; j < 3; ++j)
+          {
+            tools_WriteFloat(fp_vtk, &fXp_Act[3 * i + j], 1);
+          }
+        }
+      }
+      else
+      {
+        fprintf(fp_vtk, "%s %i %s\n", "POINTS", np_act + 1, "double");
+        for (i = 0; i <= np_act; ++i)
+        {
+          for (j = 0; j < 3; ++j)
+          {
+            tools_WriteDouble(fp_vtk, &Xp_Act[3 * i + j], 1);
+          }
+        }
+      }
     }
 
-    fprintf(fp_vtk,"%s %s %s\n","SCALARS","Patch","integer");
-    fprintf(fp_vtk,"LOOKUP_TABLE default\n");
-    for (i=0; i<cell_faces*2; ++i)
+    int vrtx = 3;
+    int* nvrtx = &vrtx;
+    fprintf(fp_vtk, "%s %i %i\n", "POLYGONS", (cell_faces) * 2, (cell_faces) * 2 * 4);
+    if (write_ascii == 1) // ASCII VTK
+    {
+      for (i = 0; i < cell_faces; ++i)
+      {
+        fprintf(fp_vtk, " %i %i %i %i\n", *nvrtx, Patch[8 * i], Patch[8 * i + 1], Patch[8 * i + 2]);
+        fprintf(fp_vtk, " %i %i %i %i\n", *nvrtx, Patch[8 * i + 3], Patch[8 * i + 4], Patch[8 * i + 5]);
+      }
+    }
+    else
     {
-      fprintf(fp_vtk,"%i \n",PatchVal[i]);
+      // BINARY VTK
+      for (i = 0; i < cell_faces; ++i)
+      {
+        tools_WriteInt(fp_vtk, nvrtx, 1);
+        for (j = 0; j < 3; ++j)
+        {
+          tools_WriteInt(fp_vtk, &Patch[8 * i + j], 1);
+        }
+        tools_WriteInt(fp_vtk, nvrtx, 1);
+        for (j = 3; j < 6; ++j)
+        {
+          tools_WriteInt(fp_vtk, &Patch[8 * i + j], 1);
+        }
+      }
     }
-  } else {
-    if (write_float)
+
+    // Now write the cell data
+    fprintf(fp_vtk, "%s %i\n", "CELL_DATA ", (cell_faces) * 2);
+    if (write_ascii == 1) // ASCII VTK
     {
-      fprintf(fp_vtk,"\n%s %s %s\n","SCALARS","Elev","float");
-      fprintf(fp_vtk,"LOOKUP_TABLE default\n");
-      for (i=0; i<cell_faces*2; ++i)
+      fprintf(fp_vtk, "%s %s %s\n", "SCALARS", "Elev", "float");
+      fprintf(fp_vtk, "LOOKUP_TABLE default\n");
+      for (i = 0; i < cell_faces * 2; ++i)
       {
-        tools_WriteFloat(fp_vtk,&fPatchEl[i], 1);
+        fprintf(fp_vtk, "%17.4f \n", PatchEl[i]);
       }
 
-      fprintf(fp_vtk,"%s %s %s\n","SCALARS","Patch","integer");
-      fprintf(fp_vtk,"LOOKUP_TABLE default\n");
-      for (i=0; i<cell_faces*2; ++i)
+      fprintf(fp_vtk, "%s %s %s\n", "SCALARS", "Patch", "integer");
+      fprintf(fp_vtk, "LOOKUP_TABLE default\n");
+      for (i = 0; i < cell_faces * 2; ++i)
       {
-        tools_WriteInt(fp_vtk,&fPatchVal[i], 1);
+        fprintf(fp_vtk, "%i \n", PatchVal[i]);
       }
-    } else
+    }
+    else
     {
-      fprintf(fp_vtk,"\n%s %s %s\n","SCALARS","Elev","double");
-      fprintf(fp_vtk,"LOOKUP_TABLE default\n");
-      for (i=0; i<cell_faces*2; ++i)
+      if (write_float)
       {
-        tools_WriteDouble(fp_vtk,&PatchEl[i], 1);
-      }
+        fprintf(fp_vtk, "\n%s %s %s\n", "SCALARS", "Elev", "float");
+        fprintf(fp_vtk, "LOOKUP_TABLE default\n");
+        for (i = 0; i < cell_faces * 2; ++i)
+        {
+          tools_WriteFloat(fp_vtk, &fPatchEl[i], 1);
+        }
 
-      fprintf(fp_vtk,"%s %s %s\n","SCALARS","Patch","integer");
-      fprintf(fp_vtk,"LOOKUP_TABLE default\n");
-      for (i=0; i<cell_faces*2; ++i)
+        fprintf(fp_vtk, "%s %s %s\n", "SCALARS", "Patch", "integer");
+        fprintf(fp_vtk, "LOOKUP_TABLE default\n");
+        for (i = 0; i < cell_faces * 2; ++i)
+        {
+          tools_WriteInt(fp_vtk, &fPatchVal[i], 1);
+        }
+      }
+      else
       {
-        tools_WriteInt(fp_vtk,&PatchVal[i], 1);
+        fprintf(fp_vtk, "\n%s %s %s\n", "SCALARS", "Elev", "double");
+        fprintf(fp_vtk, "LOOKUP_TABLE default\n");
+        for (i = 0; i < cell_faces * 2; ++i)
+        {
+          tools_WriteDouble(fp_vtk, &PatchEl[i], 1);
+        }
+
+        fprintf(fp_vtk, "%s %s %s\n", "SCALARS", "Patch", "integer");
+        fprintf(fp_vtk, "LOOKUP_TABLE default\n");
+        for (i = 0; i < cell_faces * 2; ++i)
+        {
+          tools_WriteInt(fp_vtk, &PatchVal[i], 1);
+        }
       }
     }
-  }
-
   } // End of VTK write
-  out_status=0;
+  out_status = 0;
   return out_status;
-
 }  // End of MakePatchySolid
 
 // ****************************************************************************
@@ -1124,84 +1330,92 @@ int            MakePatchySolid(
 int         ConvertPfsolAscii2Bin(FILE *fp_asc,
                                   FILE *fp_bin)
 {
-  int out_status=-1;
+  int out_status = -1;
 
-  int debug=0;
+  int debug = 0;
 
-  if (debug) printf("\n * * Debugging for Ascii2Bin converter * * \n");
+  if (debug)
+    printf("\n * * Debugging for Ascii2Bin converter * * \n");
 
-  int read_int,read_ivec[3],i,j,k;
+  int read_int, read_ivec[3], i, j, k;
   float read_fltV[3];
   double read_dble;
 
-  int n_vertex,n_tins,n_patches,n_obj;
+  int n_vertex, n_tins, n_patches, n_obj;
 
-  fscanf(fp_asc,"%i",&read_int); // Read VERSION #
-  tools_WriteInt(fp_bin,&read_int, 1); // Write VERSION #
-  if (debug) printf("\n--> Version %i \n",read_int);
+  fscanf(fp_asc, "%i", &read_int); // Read VERSION #
+  tools_WriteInt(fp_bin, &read_int, 1); // Write VERSION #
+  if (debug)
+    printf("\n--> Version %i \n", read_int);
 
-  fscanf(fp_asc,"%i",&n_vertex); // Read # of POINTS/VERTICIES
-  tools_WriteInt(fp_bin,&n_vertex, 1); //
-  if (debug) printf("--> N-Vertices %i \n",n_vertex);
+  fscanf(fp_asc, "%i", &n_vertex); // Read # of POINTS/VERTICIES
+  tools_WriteInt(fp_bin, &n_vertex, 1); //
+  if (debug)
+    printf("--> N-Vertices %i \n", n_vertex);
 
-  for (i=0; i<n_vertex; ++i)
+  for (i = 0; i < n_vertex; ++i)
   {
-    fscanf(fp_asc,"%f %f %f",&read_fltV[0],&read_fltV[1],&read_fltV[2]);
+    fscanf(fp_asc, "%f %f %f", &read_fltV[0], &read_fltV[1], &read_fltV[2]);
 
-    for (j=0; j<3; ++j)
+    for (j = 0; j < 3; ++j)
     {
-      read_dble=(double)read_fltV[j];
-      tools_WriteDouble(fp_bin,&read_dble, 1);
+      read_dble = (double)read_fltV[j];
+      tools_WriteDouble(fp_bin, &read_dble, 1);
     }
   }
 
   int nS;
-  fscanf(fp_asc,"%i",&nS); // Read Number of SOLIDS (only one allowed here)
+  fscanf(fp_asc, "%i", &nS); // Read Number of SOLIDS (only one allowed here)
   // if (nS>1)
   // {
   //   printf("ERROR (pfsolascii2asc): More than one solid object detected. Only one is allowed");
   // } else if (nS<1) {
   //   printf("ERROR (pfsolascii2asc): No one solid objects detected in pfsol");
   // }
-  tools_WriteInt(fp_bin,&nS, 1);
-  if (debug) printf("--> N-Solids %i \n",nS);
-
-  for (k=0; k<nS; ++k) {
-  fscanf(fp_asc,"%i",&n_tins); // Total number of triangles
-  tools_WriteInt(fp_bin,&n_tins, 1);
-  if (debug) printf("--> N-tins %i \n",n_tins);
+  tools_WriteInt(fp_bin, &nS, 1);
+  if (debug)
+    printf("--> N-Solids %i \n", nS);
 
-  for (i=0; i<n_tins; ++i)
+  for (k = 0; k < nS; ++k)
   {
-    fscanf(fp_asc,"%i %i %i",&read_ivec[0],&read_ivec[1],&read_ivec[2]);
-    // printf("%i %i %i\n",read_ivec[0],read_ivec[1],read_ivec[2]);
-    for (j=0; j<3; ++j)
+    fscanf(fp_asc, "%i", &n_tins); // Total number of triangles
+    tools_WriteInt(fp_bin, &n_tins, 1);
+    if (debug)
+      printf("--> N-tins %i \n", n_tins);
+
+    for (i = 0; i < n_tins; ++i)
     {
-      read_int=read_ivec[j];
-      tools_WriteInt(fp_bin,&read_int, 1);
-      // printf(" %i ",read_int);
+      fscanf(fp_asc, "%i %i %i", &read_ivec[0], &read_ivec[1], &read_ivec[2]);
+      // printf("%i %i %i\n",read_ivec[0],read_ivec[1],read_ivec[2]);
+      for (j = 0; j < 3; ++j)
+      {
+        read_int = read_ivec[j];
+        tools_WriteInt(fp_bin, &read_int, 1);
+        // printf(" %i ",read_int);
+      }
+      // printf("\n");
     }
-    // printf("\n");
-  }
 
-  fscanf(fp_asc,"%i",&n_patches); // Total number of triangles
-  tools_WriteInt(fp_bin,&n_patches, 1);
-  if (debug) printf("--> N-patch %i \n",n_patches);
+    fscanf(fp_asc, "%i", &n_patches); // Total number of triangles
+    tools_WriteInt(fp_bin, &n_patches, 1);
+    if (debug)
+      printf("--> N-patch %i \n", n_patches);
 
-  for (i=0; i<n_patches; ++i)
-  {
-    fscanf(fp_asc,"%i",&n_obj); // Elements in this patch
-    tools_WriteInt(fp_bin,&n_obj, 1);
-    if (debug) printf("--> Patch members: %i\n",n_obj);
-    for (j=0; j<n_obj; ++j)
+    for (i = 0; i < n_patches; ++i)
     {
-      fscanf(fp_asc,"%i",&read_int);
-      tools_WriteInt(fp_bin,&read_int, 1);
+      fscanf(fp_asc, "%i", &n_obj); // Elements in this patch
+      tools_WriteInt(fp_bin, &n_obj, 1);
+      if (debug)
+        printf("--> Patch members: %i\n", n_obj);
+      for (j = 0; j < n_obj; ++j)
+      {
+        fscanf(fp_asc, "%i", &read_int);
+        tools_WriteInt(fp_bin, &read_int, 1);
+      }
     }
-  }
   } // End of loop over solids
 
-  out_status=0;
+  out_status = 0;
   return out_status;
 } // End of ascii 2 binary solid file conversion
 
@@ -1214,79 +1428,87 @@ int         ConvertPfsolBin2Ascii(FILE *fp_bin,
 {
   // printf("\n ERROR (pfsolbin2ascii): Routine not enabled\n");
 
-  int out_status=-1;
+  int out_status = -1;
 
-  int debug=0;
+  int debug = 0;
 
-  if (debug) printf("\n * * Debugging for Bin2Ascii converter * * \n");
+  if (debug)
+    printf("\n * * Debugging for Bin2Ascii converter * * \n");
 
-  int read_int,read_ivec[3],i,j,k;
+  int read_int, read_ivec[3], i, j, k;
   float read_fltV[3];
   double read_dble;
 
-  int n_vertex,n_tins,n_patches,n_obj;
+  int n_vertex, n_tins, n_patches, n_obj;
 
-  tools_ReadInt(fp_bin,&read_int, 1); // Read VERSION #
-  fprintf(fp_asc,"%i\n",read_int); // Write VERSION #
-  if (debug) printf("\n--> Version %i \n",read_int);
+  tools_ReadInt(fp_bin, &read_int, 1); // Read VERSION #
+  fprintf(fp_asc, "%i\n", read_int); // Write VERSION #
+  if (debug)
+    printf("\n--> Version %i \n", read_int);
 
-  tools_ReadInt(fp_bin,&n_vertex, 1); // Read # of POINTS/VERTICIES
-  fprintf(fp_asc,"%i\n",n_vertex); //
-  if (debug) printf("--> N-Vertices %i \n",n_vertex);
-  for (i=0; i<n_vertex; ++i)
+  tools_ReadInt(fp_bin, &n_vertex, 1); // Read # of POINTS/VERTICIES
+  fprintf(fp_asc, "%i\n", n_vertex); //
+  if (debug)
+    printf("--> N-Vertices %i \n", n_vertex);
+  for (i = 0; i < n_vertex; ++i)
   {
-    for (j=0; j<3; ++j)
+    for (j = 0; j < 3; ++j)
     {
-      tools_ReadDouble(fp_bin,&read_dble, 1);
-      read_fltV[j]=read_dble;
+      tools_ReadDouble(fp_bin, &read_dble, 1);
+      read_fltV[j] = read_dble;
     }
-    fprintf(fp_asc,"%17.4f %17.4f %17.4f\n",read_fltV[0],read_fltV[1],read_fltV[2]);
+    fprintf(fp_asc, "%17.4f %17.4f %17.4f\n", read_fltV[0], read_fltV[1], read_fltV[2]);
   }
 
   int nS;
-  tools_ReadInt(fp_bin,&nS, 1); // Read Number of SOLIDS (only one allowed here)
+  tools_ReadInt(fp_bin, &nS, 1); // Read Number of SOLIDS (only one allowed here)
   // if (nS>1)
   // {
   //   printf("ERROR (pfsolascii2asc): More than one solid object detected. Only one is allowed");
   // } else if (nS<1) {
   //   printf("ERROR (pfsolascii2asc): No one solid objects detected in pfsol");
   // }
-  fprintf(fp_asc,"%i\n",nS);
-  if (debug) printf("--> N-Solids %i \n",nS);
+  fprintf(fp_asc, "%i\n", nS);
+  if (debug)
+    printf("--> N-Solids %i \n", nS);
 
-  for (k=0; k<nS; ++k) {
-  tools_ReadInt(fp_bin,&n_tins, 1); // Total number of triangles
-  fprintf(fp_asc,"%i\n",n_tins);
-  if (debug) printf("--> N-tins %i \n",n_tins);
-
-  for (i=0; i<n_tins; ++i)
+  for (k = 0; k < nS; ++k)
   {
-    for (j=0; j<3; ++j)
+    tools_ReadInt(fp_bin, &n_tins, 1); // Total number of triangles
+    fprintf(fp_asc, "%i\n", n_tins);
+    if (debug)
+      printf("--> N-tins %i \n", n_tins);
+
+    for (i = 0; i < n_tins; ++i)
     {
-      tools_ReadInt(fp_bin,&read_int, 1);
-      read_ivec[j]=read_int;
+      for (j = 0; j < 3; ++j)
+      {
+        tools_ReadInt(fp_bin, &read_int, 1);
+        read_ivec[j] = read_int;
+      }
+      fprintf(fp_asc, "%i %i %i\n", read_ivec[0], read_ivec[1], read_ivec[2]);
     }
-    fprintf(fp_asc,"%i %i %i\n",read_ivec[0],read_ivec[1],read_ivec[2]);
-  }
 
-  tools_ReadInt(fp_bin,&n_patches, 1);  // Total number of triangles
-  fprintf(fp_asc,"%i\n",n_patches);
-  if (debug) printf("--> N-patch %i \n",n_patches);
+    tools_ReadInt(fp_bin, &n_patches, 1); // Total number of triangles
+    fprintf(fp_asc, "%i\n", n_patches);
+    if (debug)
+      printf("--> N-patch %i \n", n_patches);
 
-  for (i=0; i<n_patches; ++i)
-  {
-    tools_ReadInt(fp_bin,&n_obj, 1);  // Elements in this patch
-    fprintf(fp_asc,"%i\n",n_obj);
-    if (debug) printf("--> Patch members: %i\n",n_obj);
-
-    for (j=0; j<n_obj; ++j)
+    for (i = 0; i < n_patches; ++i)
     {
-      tools_ReadInt(fp_bin,&read_int, 1);
-      fprintf(fp_asc,"%i\n",read_int);
+      tools_ReadInt(fp_bin, &n_obj, 1); // Elements in this patch
+      fprintf(fp_asc, "%i\n", n_obj);
+      if (debug)
+        printf("--> Patch members: %i\n", n_obj);
+
+      for (j = 0; j < n_obj; ++j)
+      {
+        tools_ReadInt(fp_bin, &read_int, 1);
+        fprintf(fp_asc, "%i\n", read_int);
+      }
     }
-  }
   } // End of loop over solids
 
-  out_status=0;
+  out_status = 0;
   return out_status;
 } // End of binary 2 ascii solid file conversion
diff --git a/pftools/solidtools.h b/pftools/solidtools.h
index f12cecdc0..5fa2ebe98 100644
--- a/pftools/solidtools.h
+++ b/pftools/solidtools.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Header file for `solidtools.c'
 * - Holds a variety of utilities for building and manipulating solid files
@@ -52,9 +52,9 @@ extern "C" {
  *-----------------------------------------------------------------------*/
 
 /* solidtools.c */
-int MakePatchySolid(FILE *fp,FILE *fp_vtk, Databox *msk, Databox *top, Databox *bot, int sub_patches, int bin_out);
-int ConvertPfsolBin2Ascii(FILE *fp_bin,FILE *fp_asc);
-int ConvertPfsolAscii2Bin(FILE *fp_asc,FILE *fp_bin);
+int MakePatchySolid(FILE *fp, FILE *fp_vtk, Databox *msk, Databox *top, Databox *bot, int sub_patches, int bin_out);
+int ConvertPfsolBin2Ascii(FILE *fp_bin, FILE *fp_asc);
+int ConvertPfsolAscii2Bin(FILE *fp_asc, FILE *fp_bin);
 
 #ifdef __cplusplus
 }
diff --git a/pftools/stats.c b/pftools/stats.c
index 2621738e7..a71e5c39b 100644
--- a/pftools/stats.c
+++ b/pftools/stats.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Print various statistics
 *
diff --git a/pftools/stats.h b/pftools/stats.h
index fe9759edd..2fdcf4394 100644
--- a/pftools/stats.h
+++ b/pftools/stats.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Header file for `stats.c'
 *
diff --git a/pftools/sum.c b/pftools/sum.c
index 1e9806516..3bb990d6e 100644
--- a/pftools/sum.c
+++ b/pftools/sum.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "databox.h"
 
diff --git a/pftools/tools_io.c b/pftools/tools_io.c
index ffd0d8b30..f696f7c46 100644
--- a/pftools/tools_io.c
+++ b/pftools/tools_io.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /* Code for reading/writing XDR format on non-standard systems */
 
 /* create a dummy function call to prevent warnings when compiling
diff --git a/pftools/tools_io.h b/pftools/tools_io.h
index ee196566c..b1bf6b99b 100644
--- a/pftools/tools_io.h
+++ b/pftools/tools_io.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /* Basic I/O routines */
 
 #ifndef TOOLS_IO_HEADER
@@ -110,13 +110,13 @@ void tools_ReadDouble(
 #ifdef TOOLS_INTS_ARE_64
 
 #define tools_WriteFloat(file, ptr, len) \
-  fwrite((ptr), sizeof(float), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(float), (len), (FILE*)(file))
 
 #define tools_WriteDouble(file, ptr, len) \
-  fwrite((ptr), sizeof(double), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(double), (len), (FILE*)(file))
 
 #define tools_ReadDouble(file, ptr, len) \
-  fread((ptr), sizeof(double), (len), (FILE*)(file))
+        fread((ptr), sizeof(double), (len), (FILE*)(file))
 
 #else
 
@@ -125,19 +125,19 @@ void tools_ReadDouble(
 /****************************************************************************/
 
 #define tools_WriteInt(file, ptr, len) \
-  fwrite((ptr), sizeof(int), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(int), (len), (FILE*)(file))
 
 #define tools_WriteFloat(file, ptr, len) \
-  fwrite((ptr), sizeof(float), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(float), (len), (FILE*)(file))
 
 #define tools_WriteDouble(file, ptr, len) \
-  fwrite((ptr), sizeof(double), (len), (FILE*)(file))
+        fwrite((ptr), sizeof(double), (len), (FILE*)(file))
 
 #define tools_ReadInt(file, ptr, len) \
-  fread((ptr), sizeof(int), (len), (FILE*)(file))
+        fread((ptr), sizeof(int), (len), (FILE*)(file))
 
 #define tools_ReadDouble(file, ptr, len) \
-  fread((ptr), sizeof(double), (len), (FILE*)(file))
+        fread((ptr), sizeof(double), (len), (FILE*)(file))
 
 #endif
 #endif
diff --git a/pftools/top.c b/pftools/top.c
index cd7c67da3..c4d9f2fdb 100644
--- a/pftools/top.c
+++ b/pftools/top.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "top.h"
 
 #include <stdio.h>
diff --git a/pftools/top.h b/pftools/top.h
index 16164832c..e28d9b06f 100644
--- a/pftools/top.h
+++ b/pftools/top.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #ifndef TOP_HEADER
 #define TOP_HEADER
 
diff --git a/pftools/toposlopes.c b/pftools/toposlopes.c
index 1b79a7df2..258d5a1cd 100644
--- a/pftools/toposlopes.c
+++ b/pftools/toposlopes.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "toposlopes.h"
 #include <math.h>
 #include <stdlib.h>
@@ -2845,7 +2845,7 @@ void ComputeTopoDeficitToWT(
 
   nx = DataboxNx(mask);
   ny = DataboxNy(mask);
-  
+
   dz = DataboxDz(mask);
 
   // loop over grid, skip nodata/ocean cells
@@ -3223,7 +3223,7 @@ void ComputeSegmentD8(
 
   nx = DataboxNx(dem);
   ny = DataboxNy(dem);
-  
+
   dx = DataboxDx(dem);
   dy = DataboxDy(dem);
 
diff --git a/pftools/toposlopes.h b/pftools/toposlopes.h
index 8e114ee7e..3900c7dfe 100644
--- a/pftools/toposlopes.h
+++ b/pftools/toposlopes.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #ifndef TOPOSLOPES
 #define TOPOSLOPES
 
diff --git a/pftools/usergrid.c b/pftools/usergrid.c
index 7a10dd448..0fd40408a 100644
--- a/pftools/usergrid.c
+++ b/pftools/usergrid.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "pftools.h"
 #include "general.h"
@@ -139,11 +139,11 @@ void  FreeUserGrid(
 
 #define USERGRID_MIN(x, y) ((x) < (y) ? (x) : (y))
 
-#define pqr_to_xyz(pqr, mxyz, lxyz, xyz)   (pqr*mxyz + USERGRID_MIN(pqr, lxyz) + xyz)
+#define pqr_to_xyz(pqr, mxyz, lxyz, xyz)   (pqr * mxyz + USERGRID_MIN(pqr, lxyz) + xyz)
 
 #define pqr_to_nxyz(pqr, mxyz, lxyz)  (pqr < lxyz ? mxyz + 1 : mxyz)
 
-#define pqr_to_process(p, q, r, P, Q, R)  ((((r)*(Q)) + (q))*(P) + (p))
+#define pqr_to_process(p, q, r, P, Q, R)  ((((r) * (Q)) + (q)) * (P) + (p))
 
 SubgridArray   *CopyGrid(
                          SubgridArray *all_subgrids)
diff --git a/pftools/usergrid.h b/pftools/usergrid.h
index 28b622ae7..956ba19cc 100644
--- a/pftools/usergrid.h
+++ b/pftools/usergrid.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #ifndef _USERGRID_HEADER
 #define _USERGRID_HEADER
diff --git a/pftools/velocity.c b/pftools/velocity.c
index e2729a770..af2c25eb2 100644
--- a/pftools/velocity.c
+++ b/pftools/velocity.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * CompVel, CompVMag
 *
@@ -43,10 +43,10 @@
 
 
 #if 0
-#define Mean(a, b) (0.5*((a) + (b)))
+#define Mean(a, b) (0.5 * ((a) + (b)))
 #define Mean(a, b) (sqrt((a) * (b)))
 #endif
-#define Mean(a, b) (2*((a) * (b)) / ((a) + (b)))
+#define Mean(a, b) (2 * ((a) * (b)) / ((a) + (b)))
 
 
 /*-----------------------------------------------------------------------
diff --git a/pftools/velocity.h b/pftools/velocity.h
index 135ce007c..856efe2ec 100644
--- a/pftools/velocity.h
+++ b/pftools/velocity.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 /*****************************************************************************
 * Header file for `velocity.c'
 *
diff --git a/pftools/water_balance.c b/pftools/water_balance.c
index 0432a197b..2695ffc17 100644
--- a/pftools/water_balance.c
+++ b/pftools/water_balance.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "water_balance.h"
 
 /*-----------------------------------------------------------------------
diff --git a/pftools/water_balance.h b/pftools/water_balance.h
index 5b3b3d129..edf1afa9e 100644
--- a/pftools/water_balance.h
+++ b/pftools/water_balance.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #ifndef TOP_WATER_BALANCE
 #define TOP_WATER_BALANCE
 
diff --git a/pftools/water_table.c b/pftools/water_table.c
index 6929b3e56..10b3a8444 100644
--- a/pftools/water_table.c
+++ b/pftools/water_table.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #include "water_table.h"
 
 /*-----------------------------------------------------------------------
diff --git a/pftools/water_table.h b/pftools/water_table.h
index c0285bf94..2d53a848a 100644
--- a/pftools/water_table.h
+++ b/pftools/water_table.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 #ifndef TOP_WATER_TABLE
 #define TOP_WATER_TABLE
 
diff --git a/pftools/well.c b/pftools/well.c
index ccb3381f2..5e76b8b3f 100644
--- a/pftools/well.c
+++ b/pftools/well.c
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 #include "well.h"
 
diff --git a/pftools/well.h b/pftools/well.h
index 8bfcc4378..cfe55abcb 100644
--- a/pftools/well.h
+++ b/pftools/well.h
@@ -1,30 +1,30 @@
-/*BHEADER*********************************************************************
- *
- *  Copyright (c) 1995-2009, Lawrence Livermore National Security,
- *  LLC. Produced at the Lawrence Livermore National Laboratory. Written
- *  by the Parflow Team (see the CONTRIBUTORS file)
- *  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
- *
- *  This file is part of Parflow. For details, see
- *  http://www.llnl.gov/casc/parflow
- *
- *  Please read the COPYRIGHT file or Our Notice and the LICENSE file
- *  for the GNU Lesser General Public License.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License (as published
- *  by the Free Software Foundation) version 2.1 dated February 1999.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
- *  and conditions of the GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public
- *  License along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
- *  USA
- **********************************************************************EHEADER*/
+/*BHEADER**********************************************************************
+*
+*  Copyright (c) 1995-2024, Lawrence Livermore National Security,
+*  LLC. Produced at the Lawrence Livermore National Laboratory. Written
+*  by the Parflow Team (see the CONTRIBUTORS file)
+*  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
+*
+*  This file is part of Parflow. For details, see
+*  http://www.llnl.gov/casc/parflow
+*
+*  Please read the COPYRIGHT file or Our Notice and the LICENSE file
+*  for the GNU Lesser General Public License.
+*
+*  This program is free software; you can redistribute it and/or modify
+*  it under the terms of the GNU General Public License (as published
+*  by the Free Software Foundation) version 2.1 dated February 1999.
+*
+*  This program is distributed in the hope that it will be useful, but
+*  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
+*  and conditions of the GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU Lesser General Public
+*  License along with this program; if not, write to the Free Software
+*  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+*  USA
+**********************************************************************EHEADER*/
 
 /*****************************************************************************
 * Header file for `well.c'
@@ -161,128 +161,128 @@ typedef struct {
  * Accessor macros: ProblemData
  *--------------------------------------------------------------------------*/
 #define ProblemDataNumPhases(problem_data) \
-  ((problem_data)->num_phases)
+        ((problem_data)->num_phases)
 
 #define ProblemDataNumComponents(problem_data) \
-  ((problem_data)->num_components)
+        ((problem_data)->num_components)
 
 #define ProblemDataNumWells(problem_data) \
-  ((problem_data)->num_wells)
+        ((problem_data)->num_wells)
 
 /*--------------------------------------------------------------------------
  * Accessor macros: WellDataHeader
  *--------------------------------------------------------------------------*/
 #define WellDataHeaderNumber(well_data_header) \
-  ((well_data_header)->number)
+        ((well_data_header)->number)
 
 #define WellDataHeaderName(well_data_header) \
-  ((well_data_header)->name)
+        ((well_data_header)->name)
 
 #define WellDataHeaderXLower(well_data_header) \
-  ((well_data_header)->x_lower)
+        ((well_data_header)->x_lower)
 
 #define WellDataHeaderYLower(well_data_header) \
-  ((well_data_header)->y_lower)
+        ((well_data_header)->y_lower)
 
 #define WellDataHeaderZLower(well_data_header) \
-  ((well_data_header)->z_lower)
+        ((well_data_header)->z_lower)
 
 #define WellDataHeaderXUpper(well_data_header) \
-  ((well_data_header)->x_upper)
+        ((well_data_header)->x_upper)
 
 #define WellDataHeaderYUpper(well_data_header) \
-  ((well_data_header)->y_upper)
+        ((well_data_header)->y_upper)
 
 #define WellDataHeaderZUpper(well_data_header) \
-  ((well_data_header)->z_upper)
+        ((well_data_header)->z_upper)
 
 #define WellDataHeaderDiameter(well_data_header) \
-  ((well_data_header)->diameter)
+        ((well_data_header)->diameter)
 
 #define WellDataHeaderType(well_data_header) \
-  ((well_data_header)->type)
+        ((well_data_header)->type)
 
 #define WellDataHeaderAction(well_data_header) \
-  ((well_data_header)->action)
+        ((well_data_header)->action)
 
 /*--------------------------------------------------------------------------
  * Accessor macros: WellDataPhysical
  *--------------------------------------------------------------------------*/
 #define WellDataPhysicalNumber(well_data_physical) \
-  ((well_data_physical)->number)
+        ((well_data_physical)->number)
 
 #define WellDataPhysicalIX(well_data_physical) \
-  ((well_data_physical)->ix)
+        ((well_data_physical)->ix)
 
 #define WellDataPhysicalIY(well_data_physical) \
-  ((well_data_physical)->iy)
+        ((well_data_physical)->iy)
 
 #define WellDataPhysicalIZ(well_data_physical) \
-  ((well_data_physical)->iz)
+        ((well_data_physical)->iz)
 
 #define WellDataPhysicalNX(well_data_physical) \
-  ((well_data_physical)->nx)
+        ((well_data_physical)->nx)
 
 #define WellDataPhysicalNY(well_data_physical) \
-  ((well_data_physical)->ny)
+        ((well_data_physical)->ny)
 
 #define WellDataPhysicalNZ(well_data_physical) \
-  ((well_data_physical)->nz)
+        ((well_data_physical)->nz)
 
 #define WellDataPhysicalRX(well_data_physical) \
-  ((well_data_physical)->rx)
+        ((well_data_physical)->rx)
 
 #define WellDataPhysicalRY(well_data_physical) \
-  ((well_data_physical)->ry)
+        ((well_data_physical)->ry)
 
 #define WellDataPhysicalRZ(well_data_physical) \
-  ((well_data_physical)->rz)
+        ((well_data_physical)->rz)
 
 /*--------------------------------------------------------------------------
  * Accessor macros: WellDataValue
  *--------------------------------------------------------------------------*/
 #define WellDataValuePhaseValues(well_data_value) \
-  ((well_data_value)->phase_values)
+        ((well_data_value)->phase_values)
 #define WellDataValuePhaseValue(well_data_value, i) \
-  ((well_data_value)->phase_values[i])
+        ((well_data_value)->phase_values[i])
 
 #define WellDataValueSaturationValues(well_data_value) \
-  ((well_data_value)->saturation_values)
+        ((well_data_value)->saturation_values)
 #define WellDataValueSaturationValue(well_data_value, i) \
-  ((well_data_value)->saturation_values[i])
+        ((well_data_value)->saturation_values[i])
 
 #define WellDataValueComponentValues(well_data_value) \
-  ((well_data_value)->component_values)
+        ((well_data_value)->component_values)
 #define WellDataValueComponentValue(well_data_value, i) \
-  ((well_data_value)->component_values[i])
+        ((well_data_value)->component_values[i])
 
 #define WellDataValueComponentFractions(well_data_value) \
-  ((well_data_value)->component_fractions)
+        ((well_data_value)->component_fractions)
 #define WellDataValueComponentFraction(well_data_value, i) \
-  ((well_data_value)->component_fractions[i])
+        ((well_data_value)->component_fractions[i])
 
 /*--------------------------------------------------------------------------
  * Accessor macros: WellDataStat
  *--------------------------------------------------------------------------*/
 #define WellDataStatPhaseStats(well_data_stat) \
-  ((well_data_stat)->phase_stats)
+        ((well_data_stat)->phase_stats)
 #define WellDataStatPhaseStat(well_data_stat, i) \
-  ((well_data_stat)->phase_stats[i])
+        ((well_data_stat)->phase_stats[i])
 
 #define WellDataStatSaturationStats(well_data_stat) \
-  ((well_data_stat)->saturation_stats)
+        ((well_data_stat)->saturation_stats)
 #define WellDataStatSaturationStat(well_data_stat, i) \
-  ((well_data_stat)->saturation_stats[i])
+        ((well_data_stat)->saturation_stats[i])
 
 #define WellDataStatComponentStats(well_data_stat) \
-  ((well_data_stat)->component_stats)
+        ((well_data_stat)->component_stats)
 #define WellDataStatComponentStat(well_data_stat, i) \
-  ((well_data_stat)->component_stats[i])
+        ((well_data_stat)->component_stats[i])
 
 #define WellDataStatConcentrationStats(well_data_stat) \
-  ((well_data_stat)->concentration_stats)
+        ((well_data_stat)->concentration_stats)
 #define WellDataStatConcentrationStat(well_data_stat, i) \
-  ((well_data_stat)->concentration_stats[i])
+        ((well_data_stat)->concentration_stats[i])
 
 
 /*-----------------------------------------------------------------------
diff --git a/pftools/winsync.tcl b/pftools/winsync.tcl
index 989322872..fc0904b6e 100644
--- a/pftools/winsync.tcl
+++ b/pftools/winsync.tcl
@@ -4,7 +4,7 @@ exec wish "$0" "$@"
 
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/xpfci.tcl b/pftools/xpfci.tcl
index 0a1ee28d5..8a18d0590 100644
--- a/pftools/xpfci.tcl
+++ b/pftools/xpfci.tcl
@@ -3,7 +3,7 @@
 exec wish "$0" "$@"
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/xpfmpeg.tcl b/pftools/xpfmpeg.tcl
index 535a91afd..eb9a6f8c5 100644
--- a/pftools/xpfmpeg.tcl
+++ b/pftools/xpfmpeg.tcl
@@ -4,7 +4,7 @@ exec wish "$0" "$@"
 
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/xpftdatadsp.tcl b/pftools/xpftdatadsp.tcl
index 6e2748127..92542642a 100644
--- a/pftools/xpftdatadsp.tcl
+++ b/pftools/xpftdatadsp.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/xpftdiffdsp.tcl b/pftools/xpftdiffdsp.tcl
index eb8b1dfc5..71be8a66c 100644
--- a/pftools/xpftdiffdsp.tcl
+++ b/pftools/xpftdiffdsp.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/xpftfunctions.tcl b/pftools/xpftfunctions.tcl
index c0fdc734b..43d410a92 100644
--- a/pftools/xpftfunctions.tcl
+++ b/pftools/xpftfunctions.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/xpftfuncwin.tcl b/pftools/xpftfuncwin.tcl
index f0f76b0ee..b049c637b 100644
--- a/pftools/xpftfuncwin.tcl
+++ b/pftools/xpftfuncwin.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/xpftgeneral.tcl b/pftools/xpftgeneral.tcl
index ae59d0879..5e236d22c 100644
--- a/pftools/xpftgeneral.tcl
+++ b/pftools/xpftgeneral.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/xpftgriddsp.tcl b/pftools/xpftgriddsp.tcl
index 51f6b69fe..91e1864c3 100644
--- a/pftools/xpftgriddsp.tcl
+++ b/pftools/xpftgriddsp.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/xpfthreeslices.tcl b/pftools/xpfthreeslices.tcl
index bff3e1c7a..2186aeaf7 100644
--- a/pftools/xpfthreeslices.tcl
+++ b/pftools/xpfthreeslices.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/xpftinfowin.tcl b/pftools/xpftinfowin.tcl
index 2a796c191..e60ee542e 100644
--- a/pftools/xpftinfowin.tcl
+++ b/pftools/xpftinfowin.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/xpftnewgrid.tcl b/pftools/xpftnewgrid.tcl
index ed36e3c1a..a6ae91285 100644
--- a/pftools/xpftnewgrid.tcl
+++ b/pftools/xpftnewgrid.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/xpftools.sh b/pftools/xpftools.sh
index 20b66aa23..b4f5673ab 100644
--- a/pftools/xpftools.sh
+++ b/pftools/xpftools.sh
@@ -4,7 +4,7 @@ exec wish "$0" "$@"
 
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/xpftools.tcl b/pftools/xpftools.tcl
index d2d6720b3..246f8084c 100644
--- a/pftools/xpftools.tcl
+++ b/pftools/xpftools.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/xpftpf.tcl b/pftools/xpftpf.tcl
index c5340902c..c20187d32 100644
--- a/pftools/xpftpf.tcl
+++ b/pftools/xpftpf.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/xpftsds.tcl b/pftools/xpftsds.tcl
index df9096403..923f253e0 100644
--- a/pftools/xpftsds.tcl
+++ b/pftools/xpftsds.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/pftools/xpftstatdsp.tcl b/pftools/xpftstatdsp.tcl
index bcd80cba4..89a6fd2ff 100644
--- a/pftools/xpftstatdsp.tcl
+++ b/pftools/xpftstatdsp.tcl
@@ -1,6 +1,6 @@
 #BHEADER**********************************************************************
 #
-#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
+#  Copyright (c) 1995-2024, Lawrence Livermore National Security,
 #  LLC. Produced at the Lawrence Livermore National Laboratory. Written
 #  by the Parflow Team (see the CONTRIBUTORS file)
 #  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00000.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00000.pfb
new file mode 100644
index 000000000..6193f44ac
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00000.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00001.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00001.pfb
new file mode 100644
index 000000000..01e676fa4
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00001.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00002.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00002.pfb
new file mode 100644
index 000000000..6740dd3d1
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00002.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00003.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00003.pfb
new file mode 100644
index 000000000..dc0556fe7
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00003.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00004.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00004.pfb
new file mode 100644
index 000000000..b997986b5
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00004.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00005.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00005.pfb
new file mode 100644
index 000000000..561b937a8
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00005.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00006.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00006.pfb
new file mode 100644
index 000000000..0c5c2cb7d
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00006.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00007.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00007.pfb
new file mode 100644
index 000000000..f1175654e
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00007.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00008.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00008.pfb
new file mode 100644
index 000000000..725c2acc6
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00008.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00009.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00009.pfb
new file mode 100644
index 000000000..c1291865e
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00009.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00010.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00010.pfb
new file mode 100644
index 000000000..d33f159a9
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00010.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00011.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00011.pfb
new file mode 100644
index 000000000..994f1be54
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00011.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00012.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00012.pfb
new file mode 100644
index 000000000..e3d7f8835
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00012.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00013.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00013.pfb
new file mode 100644
index 000000000..3c734380d
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00013.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00014.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00014.pfb
new file mode 100644
index 000000000..581d5f524
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00014.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00015.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00015.pfb
new file mode 100644
index 000000000..cd1525f61
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00015.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00016.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00016.pfb
new file mode 100644
index 000000000..5b8b07945
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00016.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00017.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00017.pfb
new file mode 100644
index 000000000..a1178c139
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00017.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00018.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00018.pfb
new file mode 100644
index 000000000..1caea1056
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00018.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00019.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00019.pfb
new file mode 100644
index 000000000..2b04ecba4
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00019.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00020.pfb b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00020.pfb
new file mode 100644
index 000000000..0e6b309dd
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff/TiltedV_OverlandDiff.out.press.00020.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00000.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00000.pfb
new file mode 100644
index 000000000..6193f44ac
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00000.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00001.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00001.pfb
new file mode 100644
index 000000000..01e676fa4
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00001.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00002.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00002.pfb
new file mode 100644
index 000000000..6740dd3d1
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00002.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00003.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00003.pfb
new file mode 100644
index 000000000..dc0556fe7
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00003.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00004.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00004.pfb
new file mode 100644
index 000000000..b997986b5
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00004.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00005.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00005.pfb
new file mode 100644
index 000000000..561b937a8
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00005.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00006.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00006.pfb
new file mode 100644
index 000000000..0c5c2cb7d
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00006.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00007.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00007.pfb
new file mode 100644
index 000000000..f1175654e
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00007.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00008.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00008.pfb
new file mode 100644
index 000000000..725c2acc6
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00008.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00009.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00009.pfb
new file mode 100644
index 000000000..c1291865e
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00009.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00010.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00010.pfb
new file mode 100644
index 000000000..d33f159a9
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00010.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00011.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00011.pfb
new file mode 100644
index 000000000..994f1be54
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00011.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00012.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00012.pfb
new file mode 100644
index 000000000..e3d7f8835
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00012.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00013.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00013.pfb
new file mode 100644
index 000000000..3c734380d
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00013.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00014.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00014.pfb
new file mode 100644
index 000000000..581d5f524
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00014.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00015.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00015.pfb
new file mode 100644
index 000000000..cd1525f61
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00015.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00016.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00016.pfb
new file mode 100644
index 000000000..5b8b07945
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00016.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00017.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00017.pfb
new file mode 100644
index 000000000..a1178c139
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00017.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00018.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00018.pfb
new file mode 100644
index 000000000..1caea1056
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00018.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00019.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00019.pfb
new file mode 100644
index 000000000..2b04ecba4
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00019.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00020.pfb b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00020.pfb
new file mode 100644
index 000000000..0e6b309dd
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandDiff_MGSemi/TiltedV_OverlandDiff.out.press.00020.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00000.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00000.pfb
new file mode 100644
index 000000000..6193f44ac
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00000.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00001.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00001.pfb
new file mode 100644
index 000000000..165fb22ef
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00001.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00002.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00002.pfb
new file mode 100644
index 000000000..a8049cd36
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00002.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00003.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00003.pfb
new file mode 100644
index 000000000..dea632371
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00003.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00004.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00004.pfb
new file mode 100644
index 000000000..5dfff6841
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00004.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00005.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00005.pfb
new file mode 100644
index 000000000..a0dd6b442
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00005.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00006.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00006.pfb
new file mode 100644
index 000000000..a55a8be17
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00006.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00007.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00007.pfb
new file mode 100644
index 000000000..45154356f
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00007.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00008.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00008.pfb
new file mode 100644
index 000000000..bc9a4ee5a
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00008.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00009.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00009.pfb
new file mode 100644
index 000000000..bc20ec35e
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00009.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00010.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00010.pfb
new file mode 100644
index 000000000..4f0ce7aa3
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00010.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00011.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00011.pfb
new file mode 100644
index 000000000..30f67bba5
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00011.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00012.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00012.pfb
new file mode 100644
index 000000000..1d92c2491
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00012.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00013.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00013.pfb
new file mode 100644
index 000000000..9618c1359
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00013.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00014.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00014.pfb
new file mode 100644
index 000000000..7e930d6b6
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00014.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00015.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00015.pfb
new file mode 100644
index 000000000..5379be96f
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00015.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00016.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00016.pfb
new file mode 100644
index 000000000..0d5dd994e
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00016.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00017.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00017.pfb
new file mode 100644
index 000000000..06b5405e0
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00017.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00018.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00018.pfb
new file mode 100644
index 000000000..32ff48a69
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00018.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00019.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00019.pfb
new file mode 100644
index 000000000..fc2eb550b
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00019.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00020.pfb b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00020.pfb
new file mode 100644
index 000000000..c41b866f9
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow/TiltedV_OverlandFlow.out.press.00020.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00000.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00000.pfb
new file mode 100644
index 000000000..6193f44ac
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00000.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00001.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00001.pfb
new file mode 100644
index 000000000..165fb22ef
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00001.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00002.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00002.pfb
new file mode 100644
index 000000000..a8049cd36
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00002.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00003.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00003.pfb
new file mode 100644
index 000000000..dea632371
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00003.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00004.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00004.pfb
new file mode 100644
index 000000000..5dfff6841
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00004.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00005.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00005.pfb
new file mode 100644
index 000000000..a0dd6b442
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00005.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00006.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00006.pfb
new file mode 100644
index 000000000..a55a8be17
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00006.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00007.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00007.pfb
new file mode 100644
index 000000000..45154356f
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00007.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00008.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00008.pfb
new file mode 100644
index 000000000..bc9a4ee5a
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00008.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00009.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00009.pfb
new file mode 100644
index 000000000..bc20ec35e
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00009.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00010.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00010.pfb
new file mode 100644
index 000000000..4f0ce7aa3
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00010.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00011.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00011.pfb
new file mode 100644
index 000000000..30f67bba5
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00011.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00012.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00012.pfb
new file mode 100644
index 000000000..1d92c2491
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00012.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00013.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00013.pfb
new file mode 100644
index 000000000..9618c1359
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00013.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00014.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00014.pfb
new file mode 100644
index 000000000..7e930d6b6
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00014.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00015.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00015.pfb
new file mode 100644
index 000000000..5379be96f
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00015.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00016.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00016.pfb
new file mode 100644
index 000000000..0d5dd994e
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00016.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00017.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00017.pfb
new file mode 100644
index 000000000..06b5405e0
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00017.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00018.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00018.pfb
new file mode 100644
index 000000000..32ff48a69
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00018.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00019.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00019.pfb
new file mode 100644
index 000000000..fc2eb550b
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00019.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00020.pfb b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00020.pfb
new file mode 100644
index 000000000..c41b866f9
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandFlow_MGSemi/TiltedV_OverlandFlow.out.press.00020.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00000.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00000.pfb
new file mode 100644
index 000000000..6193f44ac
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00000.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00001.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00001.pfb
new file mode 100644
index 000000000..06cb25685
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00001.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00002.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00002.pfb
new file mode 100644
index 000000000..5ac2567b2
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00002.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00003.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00003.pfb
new file mode 100644
index 000000000..cd081faa3
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00003.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00004.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00004.pfb
new file mode 100644
index 000000000..bf5cf7850
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00004.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00005.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00005.pfb
new file mode 100644
index 000000000..99b23076a
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00005.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00006.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00006.pfb
new file mode 100644
index 000000000..ffce8a011
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00006.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00007.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00007.pfb
new file mode 100644
index 000000000..b6abd48ab
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00007.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00008.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00008.pfb
new file mode 100644
index 000000000..9d4bcbb8b
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00008.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00009.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00009.pfb
new file mode 100644
index 000000000..667a05c57
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00009.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00010.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00010.pfb
new file mode 100644
index 000000000..c946b221e
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00010.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00011.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00011.pfb
new file mode 100644
index 000000000..6950a0ab6
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00011.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00012.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00012.pfb
new file mode 100644
index 000000000..e5d1dad14
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00012.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00013.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00013.pfb
new file mode 100644
index 000000000..bb4cb5ad6
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00013.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00014.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00014.pfb
new file mode 100644
index 000000000..0cb6758cb
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00014.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00015.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00015.pfb
new file mode 100644
index 000000000..e613a0ce3
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00015.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00016.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00016.pfb
new file mode 100644
index 000000000..3d328bfc3
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00016.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00017.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00017.pfb
new file mode 100644
index 000000000..c118b4ae5
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00017.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00018.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00018.pfb
new file mode 100644
index 000000000..d956127ed
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00018.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00019.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00019.pfb
new file mode 100644
index 000000000..66c38d385
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00019.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00020.pfb b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00020.pfb
new file mode 100644
index 000000000..1afb43bba
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin/TiltedV_OverlandKin.out.press.00020.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00000.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00000.pfb
new file mode 100644
index 000000000..6193f44ac
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00000.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00001.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00001.pfb
new file mode 100644
index 000000000..06cb25685
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00001.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00002.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00002.pfb
new file mode 100644
index 000000000..5ac2567b2
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00002.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00003.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00003.pfb
new file mode 100644
index 000000000..cd081faa3
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00003.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00004.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00004.pfb
new file mode 100644
index 000000000..bf5cf7850
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00004.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00005.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00005.pfb
new file mode 100644
index 000000000..99b23076a
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00005.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00006.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00006.pfb
new file mode 100644
index 000000000..ffce8a011
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00006.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00007.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00007.pfb
new file mode 100644
index 000000000..b6abd48ab
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00007.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00008.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00008.pfb
new file mode 100644
index 000000000..9d4bcbb8b
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00008.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00009.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00009.pfb
new file mode 100644
index 000000000..667a05c57
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00009.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00010.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00010.pfb
new file mode 100644
index 000000000..c946b221e
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00010.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00011.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00011.pfb
new file mode 100644
index 000000000..6950a0ab6
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00011.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00012.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00012.pfb
new file mode 100644
index 000000000..e5d1dad14
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00012.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00013.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00013.pfb
new file mode 100644
index 000000000..bb4cb5ad6
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00013.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00014.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00014.pfb
new file mode 100644
index 000000000..0cb6758cb
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00014.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00015.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00015.pfb
new file mode 100644
index 000000000..e613a0ce3
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00015.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00016.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00016.pfb
new file mode 100644
index 000000000..3d328bfc3
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00016.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00017.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00017.pfb
new file mode 100644
index 000000000..c118b4ae5
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00017.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00018.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00018.pfb
new file mode 100644
index 000000000..d956127ed
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00018.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00019.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00019.pfb
new file mode 100644
index 000000000..66c38d385
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00019.pfb differ
diff --git a/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00020.pfb b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00020.pfb
new file mode 100644
index 000000000..1afb43bba
Binary files /dev/null and b/test/correct_output/TiltedV_OverlandKin_MGSemi/TiltedV_OverlandKin.out.press.00020.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00002.pfb b/test/correct_output/default_richards.out.press.00002.pfb
index 6966af7a3..46409da4d 100644
Binary files a/test/correct_output/default_richards.out.press.00002.pfb and b/test/correct_output/default_richards.out.press.00002.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00003.pfb b/test/correct_output/default_richards.out.press.00003.pfb
index 6c642eab9..bc023bc64 100644
Binary files a/test/correct_output/default_richards.out.press.00003.pfb and b/test/correct_output/default_richards.out.press.00003.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00004.pfb b/test/correct_output/default_richards.out.press.00004.pfb
index 4f1b50791..72030f072 100644
Binary files a/test/correct_output/default_richards.out.press.00004.pfb and b/test/correct_output/default_richards.out.press.00004.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00005.pfb b/test/correct_output/default_richards.out.press.00005.pfb
index fd2f2dc34..926c4987c 100644
Binary files a/test/correct_output/default_richards.out.press.00005.pfb and b/test/correct_output/default_richards.out.press.00005.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00006.pfb b/test/correct_output/default_richards.out.press.00006.pfb
new file mode 100644
index 000000000..446e257e0
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00006.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00007.pfb b/test/correct_output/default_richards.out.press.00007.pfb
new file mode 100644
index 000000000..bad289d6c
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00007.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00008.pfb b/test/correct_output/default_richards.out.press.00008.pfb
new file mode 100644
index 000000000..75b90db8d
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00008.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00009.pfb b/test/correct_output/default_richards.out.press.00009.pfb
new file mode 100644
index 000000000..ec1940d23
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00009.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00010.pfb b/test/correct_output/default_richards.out.press.00010.pfb
new file mode 100644
index 000000000..83b63f987
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00010.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00011.pfb b/test/correct_output/default_richards.out.press.00011.pfb
new file mode 100644
index 000000000..2cdd991a0
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00011.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00012.pfb b/test/correct_output/default_richards.out.press.00012.pfb
new file mode 100644
index 000000000..d869dcaae
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00012.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00013.pfb b/test/correct_output/default_richards.out.press.00013.pfb
new file mode 100644
index 000000000..e086dbce3
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00013.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00014.pfb b/test/correct_output/default_richards.out.press.00014.pfb
new file mode 100644
index 000000000..e48878912
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00014.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00015.pfb b/test/correct_output/default_richards.out.press.00015.pfb
new file mode 100644
index 000000000..a79a20bc7
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00015.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00016.pfb b/test/correct_output/default_richards.out.press.00016.pfb
new file mode 100644
index 000000000..b7e976c07
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00016.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00017.pfb b/test/correct_output/default_richards.out.press.00017.pfb
new file mode 100644
index 000000000..20f215a10
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00017.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00018.pfb b/test/correct_output/default_richards.out.press.00018.pfb
new file mode 100644
index 000000000..be39cd26a
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00018.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00019.pfb b/test/correct_output/default_richards.out.press.00019.pfb
new file mode 100644
index 000000000..a7da39d3c
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00019.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00020.pfb b/test/correct_output/default_richards.out.press.00020.pfb
new file mode 100644
index 000000000..94c18fbf3
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00020.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00021.pfb b/test/correct_output/default_richards.out.press.00021.pfb
new file mode 100644
index 000000000..50b058c24
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00021.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00022.pfb b/test/correct_output/default_richards.out.press.00022.pfb
new file mode 100644
index 000000000..66e415bf8
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00022.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00023.pfb b/test/correct_output/default_richards.out.press.00023.pfb
new file mode 100644
index 000000000..ac65e2f27
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00023.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00024.pfb b/test/correct_output/default_richards.out.press.00024.pfb
new file mode 100644
index 000000000..28384ea86
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00024.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00025.pfb b/test/correct_output/default_richards.out.press.00025.pfb
new file mode 100644
index 000000000..933936218
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00025.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00026.pfb b/test/correct_output/default_richards.out.press.00026.pfb
new file mode 100644
index 000000000..35b4868f0
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00026.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00027.pfb b/test/correct_output/default_richards.out.press.00027.pfb
new file mode 100644
index 000000000..6e32c6123
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00027.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00028.pfb b/test/correct_output/default_richards.out.press.00028.pfb
new file mode 100644
index 000000000..7276f19da
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00028.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00029.pfb b/test/correct_output/default_richards.out.press.00029.pfb
new file mode 100644
index 000000000..138ce7d43
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00029.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00030.pfb b/test/correct_output/default_richards.out.press.00030.pfb
new file mode 100644
index 000000000..1834620e1
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00030.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00031.pfb b/test/correct_output/default_richards.out.press.00031.pfb
new file mode 100644
index 000000000..ca7a39b52
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00031.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00032.pfb b/test/correct_output/default_richards.out.press.00032.pfb
new file mode 100644
index 000000000..3f8a3c797
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00032.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00033.pfb b/test/correct_output/default_richards.out.press.00033.pfb
new file mode 100644
index 000000000..9895b2b1a
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00033.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00034.pfb b/test/correct_output/default_richards.out.press.00034.pfb
new file mode 100644
index 000000000..4b0b8dc38
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00034.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00035.pfb b/test/correct_output/default_richards.out.press.00035.pfb
new file mode 100644
index 000000000..4a649282d
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00035.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00036.pfb b/test/correct_output/default_richards.out.press.00036.pfb
new file mode 100644
index 000000000..548a495b5
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00036.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00037.pfb b/test/correct_output/default_richards.out.press.00037.pfb
new file mode 100644
index 000000000..c4dcc8831
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00037.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00038.pfb b/test/correct_output/default_richards.out.press.00038.pfb
new file mode 100644
index 000000000..101f61149
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00038.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00039.pfb b/test/correct_output/default_richards.out.press.00039.pfb
new file mode 100644
index 000000000..4391793de
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00039.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00040.pfb b/test/correct_output/default_richards.out.press.00040.pfb
new file mode 100644
index 000000000..72ce7badc
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00040.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00041.pfb b/test/correct_output/default_richards.out.press.00041.pfb
new file mode 100644
index 000000000..a9dbef192
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00041.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00042.pfb b/test/correct_output/default_richards.out.press.00042.pfb
new file mode 100644
index 000000000..fbe6251df
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00042.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00043.pfb b/test/correct_output/default_richards.out.press.00043.pfb
new file mode 100644
index 000000000..b50f2b984
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00043.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00044.pfb b/test/correct_output/default_richards.out.press.00044.pfb
new file mode 100644
index 000000000..fc9fec318
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00044.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00045.pfb b/test/correct_output/default_richards.out.press.00045.pfb
new file mode 100644
index 000000000..ac26dce48
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00045.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00046.pfb b/test/correct_output/default_richards.out.press.00046.pfb
new file mode 100644
index 000000000..9f7cdcb70
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00046.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00047.pfb b/test/correct_output/default_richards.out.press.00047.pfb
new file mode 100644
index 000000000..d430f4fcf
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00047.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00048.pfb b/test/correct_output/default_richards.out.press.00048.pfb
new file mode 100644
index 000000000..962762ce9
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00048.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00049.pfb b/test/correct_output/default_richards.out.press.00049.pfb
new file mode 100644
index 000000000..4646a8fd0
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00049.pfb differ
diff --git a/test/correct_output/default_richards.out.press.00050.pfb b/test/correct_output/default_richards.out.press.00050.pfb
new file mode 100644
index 000000000..018db8944
Binary files /dev/null and b/test/correct_output/default_richards.out.press.00050.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00006.pfb b/test/correct_output/default_richards.out.satur.00006.pfb
new file mode 100644
index 000000000..b398007b7
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00006.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00007.pfb b/test/correct_output/default_richards.out.satur.00007.pfb
new file mode 100644
index 000000000..04305a0bf
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00007.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00008.pfb b/test/correct_output/default_richards.out.satur.00008.pfb
new file mode 100644
index 000000000..2be6fe392
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00008.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00009.pfb b/test/correct_output/default_richards.out.satur.00009.pfb
new file mode 100644
index 000000000..bb58d67bf
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00009.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00010.pfb b/test/correct_output/default_richards.out.satur.00010.pfb
new file mode 100644
index 000000000..b586d2abf
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00010.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00011.pfb b/test/correct_output/default_richards.out.satur.00011.pfb
new file mode 100644
index 000000000..a39a026b8
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00011.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00012.pfb b/test/correct_output/default_richards.out.satur.00012.pfb
new file mode 100644
index 000000000..e4575cc0e
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00012.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00013.pfb b/test/correct_output/default_richards.out.satur.00013.pfb
new file mode 100644
index 000000000..3d5e1201d
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00013.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00014.pfb b/test/correct_output/default_richards.out.satur.00014.pfb
new file mode 100644
index 000000000..df514b10e
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00014.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00015.pfb b/test/correct_output/default_richards.out.satur.00015.pfb
new file mode 100644
index 000000000..e06e4fc27
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00015.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00016.pfb b/test/correct_output/default_richards.out.satur.00016.pfb
new file mode 100644
index 000000000..d301ecd9a
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00016.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00017.pfb b/test/correct_output/default_richards.out.satur.00017.pfb
new file mode 100644
index 000000000..395b3beef
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00017.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00018.pfb b/test/correct_output/default_richards.out.satur.00018.pfb
new file mode 100644
index 000000000..fcea60a18
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00018.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00019.pfb b/test/correct_output/default_richards.out.satur.00019.pfb
new file mode 100644
index 000000000..03d9f06f6
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00019.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00020.pfb b/test/correct_output/default_richards.out.satur.00020.pfb
new file mode 100644
index 000000000..903e87909
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00020.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00021.pfb b/test/correct_output/default_richards.out.satur.00021.pfb
new file mode 100644
index 000000000..fd3ec3ccf
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00021.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00022.pfb b/test/correct_output/default_richards.out.satur.00022.pfb
new file mode 100644
index 000000000..c03e0196f
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00022.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00023.pfb b/test/correct_output/default_richards.out.satur.00023.pfb
new file mode 100644
index 000000000..776878e8e
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00023.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00024.pfb b/test/correct_output/default_richards.out.satur.00024.pfb
new file mode 100644
index 000000000..9a2c73b20
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00024.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00025.pfb b/test/correct_output/default_richards.out.satur.00025.pfb
new file mode 100644
index 000000000..0ad5d3192
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00025.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00026.pfb b/test/correct_output/default_richards.out.satur.00026.pfb
new file mode 100644
index 000000000..fd404891f
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00026.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00027.pfb b/test/correct_output/default_richards.out.satur.00027.pfb
new file mode 100644
index 000000000..25f8ad267
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00027.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00028.pfb b/test/correct_output/default_richards.out.satur.00028.pfb
new file mode 100644
index 000000000..ab62b4a81
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00028.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00029.pfb b/test/correct_output/default_richards.out.satur.00029.pfb
new file mode 100644
index 000000000..5659eff6d
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00029.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00030.pfb b/test/correct_output/default_richards.out.satur.00030.pfb
new file mode 100644
index 000000000..2d5010978
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00030.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00031.pfb b/test/correct_output/default_richards.out.satur.00031.pfb
new file mode 100644
index 000000000..231d51ca2
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00031.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00032.pfb b/test/correct_output/default_richards.out.satur.00032.pfb
new file mode 100644
index 000000000..84d871918
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00032.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00033.pfb b/test/correct_output/default_richards.out.satur.00033.pfb
new file mode 100644
index 000000000..7eb8b7806
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00033.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00034.pfb b/test/correct_output/default_richards.out.satur.00034.pfb
new file mode 100644
index 000000000..74b9b3b59
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00034.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00035.pfb b/test/correct_output/default_richards.out.satur.00035.pfb
new file mode 100644
index 000000000..0efd6f096
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00035.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00036.pfb b/test/correct_output/default_richards.out.satur.00036.pfb
new file mode 100644
index 000000000..04b70628a
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00036.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00037.pfb b/test/correct_output/default_richards.out.satur.00037.pfb
new file mode 100644
index 000000000..77d1f055a
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00037.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00038.pfb b/test/correct_output/default_richards.out.satur.00038.pfb
new file mode 100644
index 000000000..5d68db324
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00038.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00039.pfb b/test/correct_output/default_richards.out.satur.00039.pfb
new file mode 100644
index 000000000..d40770bba
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00039.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00040.pfb b/test/correct_output/default_richards.out.satur.00040.pfb
new file mode 100644
index 000000000..32faf889e
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00040.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00041.pfb b/test/correct_output/default_richards.out.satur.00041.pfb
new file mode 100644
index 000000000..7ed88c1f8
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00041.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00042.pfb b/test/correct_output/default_richards.out.satur.00042.pfb
new file mode 100644
index 000000000..8e278b9a0
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00042.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00043.pfb b/test/correct_output/default_richards.out.satur.00043.pfb
new file mode 100644
index 000000000..5950ffe14
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00043.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00044.pfb b/test/correct_output/default_richards.out.satur.00044.pfb
new file mode 100644
index 000000000..54d40596e
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00044.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00045.pfb b/test/correct_output/default_richards.out.satur.00045.pfb
new file mode 100644
index 000000000..6a20b69f9
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00045.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00046.pfb b/test/correct_output/default_richards.out.satur.00046.pfb
new file mode 100644
index 000000000..a74a5d7d6
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00046.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00047.pfb b/test/correct_output/default_richards.out.satur.00047.pfb
new file mode 100644
index 000000000..d07e00e1d
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00047.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00048.pfb b/test/correct_output/default_richards.out.satur.00048.pfb
new file mode 100644
index 000000000..4596a8087
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00048.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00049.pfb b/test/correct_output/default_richards.out.satur.00049.pfb
new file mode 100644
index 000000000..886da7ed1
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00049.pfb differ
diff --git a/test/correct_output/default_richards.out.satur.00050.pfb b/test/correct_output/default_richards.out.satur.00050.pfb
new file mode 100644
index 000000000..ec7bd7863
Binary files /dev/null and b/test/correct_output/default_richards.out.satur.00050.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00001.pfb b/test/correct_output/default_richards.out.velx.00001.pfb
index a2c9a3bb1..4ad715dbf 100644
Binary files a/test/correct_output/default_richards.out.velx.00001.pfb and b/test/correct_output/default_richards.out.velx.00001.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00002.pfb b/test/correct_output/default_richards.out.velx.00002.pfb
index b7e614c56..4675873f1 100644
Binary files a/test/correct_output/default_richards.out.velx.00002.pfb and b/test/correct_output/default_richards.out.velx.00002.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00003.pfb b/test/correct_output/default_richards.out.velx.00003.pfb
index ff293e9fb..59443a266 100644
Binary files a/test/correct_output/default_richards.out.velx.00003.pfb and b/test/correct_output/default_richards.out.velx.00003.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00004.pfb b/test/correct_output/default_richards.out.velx.00004.pfb
index 6c2f3f22f..1c6083767 100644
Binary files a/test/correct_output/default_richards.out.velx.00004.pfb and b/test/correct_output/default_richards.out.velx.00004.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00005.pfb b/test/correct_output/default_richards.out.velx.00005.pfb
index 099953c59..5ed297d61 100644
Binary files a/test/correct_output/default_richards.out.velx.00005.pfb and b/test/correct_output/default_richards.out.velx.00005.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00006.pfb b/test/correct_output/default_richards.out.velx.00006.pfb
new file mode 100644
index 000000000..7a35fd29c
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00006.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00007.pfb b/test/correct_output/default_richards.out.velx.00007.pfb
new file mode 100644
index 000000000..792eea7cc
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00007.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00008.pfb b/test/correct_output/default_richards.out.velx.00008.pfb
new file mode 100644
index 000000000..324d059d9
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00008.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00009.pfb b/test/correct_output/default_richards.out.velx.00009.pfb
new file mode 100644
index 000000000..de5d6e58d
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00009.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00010.pfb b/test/correct_output/default_richards.out.velx.00010.pfb
new file mode 100644
index 000000000..35d9c9f2e
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00010.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00011.pfb b/test/correct_output/default_richards.out.velx.00011.pfb
new file mode 100644
index 000000000..da7c50919
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00011.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00012.pfb b/test/correct_output/default_richards.out.velx.00012.pfb
new file mode 100644
index 000000000..42d310dcc
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00012.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00013.pfb b/test/correct_output/default_richards.out.velx.00013.pfb
new file mode 100644
index 000000000..df545c74b
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00013.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00014.pfb b/test/correct_output/default_richards.out.velx.00014.pfb
new file mode 100644
index 000000000..14be1a1e1
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00014.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00015.pfb b/test/correct_output/default_richards.out.velx.00015.pfb
new file mode 100644
index 000000000..1a8be5091
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00015.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00016.pfb b/test/correct_output/default_richards.out.velx.00016.pfb
new file mode 100644
index 000000000..857231051
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00016.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00017.pfb b/test/correct_output/default_richards.out.velx.00017.pfb
new file mode 100644
index 000000000..69eeac9ee
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00017.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00018.pfb b/test/correct_output/default_richards.out.velx.00018.pfb
new file mode 100644
index 000000000..50b58eace
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00018.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00019.pfb b/test/correct_output/default_richards.out.velx.00019.pfb
new file mode 100644
index 000000000..6480c266f
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00019.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00020.pfb b/test/correct_output/default_richards.out.velx.00020.pfb
new file mode 100644
index 000000000..2f8055842
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00020.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00021.pfb b/test/correct_output/default_richards.out.velx.00021.pfb
new file mode 100644
index 000000000..e0ea6b1b7
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00021.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00022.pfb b/test/correct_output/default_richards.out.velx.00022.pfb
new file mode 100644
index 000000000..3b05ce6ad
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00022.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00023.pfb b/test/correct_output/default_richards.out.velx.00023.pfb
new file mode 100644
index 000000000..e7016af50
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00023.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00024.pfb b/test/correct_output/default_richards.out.velx.00024.pfb
new file mode 100644
index 000000000..d0dc8a9cd
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00024.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00025.pfb b/test/correct_output/default_richards.out.velx.00025.pfb
new file mode 100644
index 000000000..b4f2882ba
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00025.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00026.pfb b/test/correct_output/default_richards.out.velx.00026.pfb
new file mode 100644
index 000000000..b532b5845
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00026.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00027.pfb b/test/correct_output/default_richards.out.velx.00027.pfb
new file mode 100644
index 000000000..26c190990
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00027.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00028.pfb b/test/correct_output/default_richards.out.velx.00028.pfb
new file mode 100644
index 000000000..563909d2e
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00028.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00029.pfb b/test/correct_output/default_richards.out.velx.00029.pfb
new file mode 100644
index 000000000..efe67d2ab
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00029.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00030.pfb b/test/correct_output/default_richards.out.velx.00030.pfb
new file mode 100644
index 000000000..71997717a
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00030.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00031.pfb b/test/correct_output/default_richards.out.velx.00031.pfb
new file mode 100644
index 000000000..a100f232c
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00031.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00032.pfb b/test/correct_output/default_richards.out.velx.00032.pfb
new file mode 100644
index 000000000..295dcdeef
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00032.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00033.pfb b/test/correct_output/default_richards.out.velx.00033.pfb
new file mode 100644
index 000000000..3760e1c2d
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00033.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00034.pfb b/test/correct_output/default_richards.out.velx.00034.pfb
new file mode 100644
index 000000000..7b9ad0578
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00034.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00035.pfb b/test/correct_output/default_richards.out.velx.00035.pfb
new file mode 100644
index 000000000..bea6411b9
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00035.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00036.pfb b/test/correct_output/default_richards.out.velx.00036.pfb
new file mode 100644
index 000000000..646868c03
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00036.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00037.pfb b/test/correct_output/default_richards.out.velx.00037.pfb
new file mode 100644
index 000000000..0b384b96f
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00037.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00038.pfb b/test/correct_output/default_richards.out.velx.00038.pfb
new file mode 100644
index 000000000..36d29a990
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00038.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00039.pfb b/test/correct_output/default_richards.out.velx.00039.pfb
new file mode 100644
index 000000000..46406c7cd
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00039.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00040.pfb b/test/correct_output/default_richards.out.velx.00040.pfb
new file mode 100644
index 000000000..897c011a7
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00040.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00041.pfb b/test/correct_output/default_richards.out.velx.00041.pfb
new file mode 100644
index 000000000..e0e220e32
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00041.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00042.pfb b/test/correct_output/default_richards.out.velx.00042.pfb
new file mode 100644
index 000000000..424bd6b52
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00042.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00043.pfb b/test/correct_output/default_richards.out.velx.00043.pfb
new file mode 100644
index 000000000..eda79edce
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00043.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00044.pfb b/test/correct_output/default_richards.out.velx.00044.pfb
new file mode 100644
index 000000000..99a25b2d8
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00044.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00045.pfb b/test/correct_output/default_richards.out.velx.00045.pfb
new file mode 100644
index 000000000..058a5e51d
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00045.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00046.pfb b/test/correct_output/default_richards.out.velx.00046.pfb
new file mode 100644
index 000000000..b02ee43e4
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00046.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00047.pfb b/test/correct_output/default_richards.out.velx.00047.pfb
new file mode 100644
index 000000000..f303c0c25
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00047.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00048.pfb b/test/correct_output/default_richards.out.velx.00048.pfb
new file mode 100644
index 000000000..f1e753d62
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00048.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00049.pfb b/test/correct_output/default_richards.out.velx.00049.pfb
new file mode 100644
index 000000000..0e03cea59
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00049.pfb differ
diff --git a/test/correct_output/default_richards.out.velx.00050.pfb b/test/correct_output/default_richards.out.velx.00050.pfb
new file mode 100644
index 000000000..dcb9ae612
Binary files /dev/null and b/test/correct_output/default_richards.out.velx.00050.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00001.pfb b/test/correct_output/default_richards.out.vely.00001.pfb
index 050eecdf3..5f25e656f 100644
Binary files a/test/correct_output/default_richards.out.vely.00001.pfb and b/test/correct_output/default_richards.out.vely.00001.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00002.pfb b/test/correct_output/default_richards.out.vely.00002.pfb
index 554a05ec2..e2e764fd0 100644
Binary files a/test/correct_output/default_richards.out.vely.00002.pfb and b/test/correct_output/default_richards.out.vely.00002.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00003.pfb b/test/correct_output/default_richards.out.vely.00003.pfb
index 152b6bf19..48efd55cf 100644
Binary files a/test/correct_output/default_richards.out.vely.00003.pfb and b/test/correct_output/default_richards.out.vely.00003.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00004.pfb b/test/correct_output/default_richards.out.vely.00004.pfb
index 2bb3a84f5..41c903daf 100644
Binary files a/test/correct_output/default_richards.out.vely.00004.pfb and b/test/correct_output/default_richards.out.vely.00004.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00005.pfb b/test/correct_output/default_richards.out.vely.00005.pfb
index 9577ecb2f..a378d4658 100644
Binary files a/test/correct_output/default_richards.out.vely.00005.pfb and b/test/correct_output/default_richards.out.vely.00005.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00006.pfb b/test/correct_output/default_richards.out.vely.00006.pfb
new file mode 100644
index 000000000..0446dcba9
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00006.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00007.pfb b/test/correct_output/default_richards.out.vely.00007.pfb
new file mode 100644
index 000000000..a718b0de4
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00007.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00008.pfb b/test/correct_output/default_richards.out.vely.00008.pfb
new file mode 100644
index 000000000..1d5517e74
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00008.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00009.pfb b/test/correct_output/default_richards.out.vely.00009.pfb
new file mode 100644
index 000000000..1007153f4
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00009.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00010.pfb b/test/correct_output/default_richards.out.vely.00010.pfb
new file mode 100644
index 000000000..a08fac242
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00010.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00011.pfb b/test/correct_output/default_richards.out.vely.00011.pfb
new file mode 100644
index 000000000..657b897aa
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00011.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00012.pfb b/test/correct_output/default_richards.out.vely.00012.pfb
new file mode 100644
index 000000000..f328dc7dc
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00012.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00013.pfb b/test/correct_output/default_richards.out.vely.00013.pfb
new file mode 100644
index 000000000..e498a0819
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00013.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00014.pfb b/test/correct_output/default_richards.out.vely.00014.pfb
new file mode 100644
index 000000000..dac124557
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00014.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00015.pfb b/test/correct_output/default_richards.out.vely.00015.pfb
new file mode 100644
index 000000000..72151bd3b
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00015.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00016.pfb b/test/correct_output/default_richards.out.vely.00016.pfb
new file mode 100644
index 000000000..359c3eb37
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00016.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00017.pfb b/test/correct_output/default_richards.out.vely.00017.pfb
new file mode 100644
index 000000000..b2f872359
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00017.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00018.pfb b/test/correct_output/default_richards.out.vely.00018.pfb
new file mode 100644
index 000000000..c79db134a
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00018.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00019.pfb b/test/correct_output/default_richards.out.vely.00019.pfb
new file mode 100644
index 000000000..613e1364b
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00019.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00020.pfb b/test/correct_output/default_richards.out.vely.00020.pfb
new file mode 100644
index 000000000..edb50e465
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00020.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00021.pfb b/test/correct_output/default_richards.out.vely.00021.pfb
new file mode 100644
index 000000000..53954ad9d
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00021.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00022.pfb b/test/correct_output/default_richards.out.vely.00022.pfb
new file mode 100644
index 000000000..5e210b9ab
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00022.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00023.pfb b/test/correct_output/default_richards.out.vely.00023.pfb
new file mode 100644
index 000000000..3e7db6d62
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00023.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00024.pfb b/test/correct_output/default_richards.out.vely.00024.pfb
new file mode 100644
index 000000000..ae92fb587
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00024.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00025.pfb b/test/correct_output/default_richards.out.vely.00025.pfb
new file mode 100644
index 000000000..0aa98e596
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00025.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00026.pfb b/test/correct_output/default_richards.out.vely.00026.pfb
new file mode 100644
index 000000000..2d8b40eee
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00026.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00027.pfb b/test/correct_output/default_richards.out.vely.00027.pfb
new file mode 100644
index 000000000..e2d3f9d5b
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00027.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00028.pfb b/test/correct_output/default_richards.out.vely.00028.pfb
new file mode 100644
index 000000000..b180b0755
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00028.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00029.pfb b/test/correct_output/default_richards.out.vely.00029.pfb
new file mode 100644
index 000000000..b4c5a74c3
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00029.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00030.pfb b/test/correct_output/default_richards.out.vely.00030.pfb
new file mode 100644
index 000000000..470c40510
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00030.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00031.pfb b/test/correct_output/default_richards.out.vely.00031.pfb
new file mode 100644
index 000000000..2da340e63
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00031.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00032.pfb b/test/correct_output/default_richards.out.vely.00032.pfb
new file mode 100644
index 000000000..53c453344
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00032.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00033.pfb b/test/correct_output/default_richards.out.vely.00033.pfb
new file mode 100644
index 000000000..71b5700f5
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00033.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00034.pfb b/test/correct_output/default_richards.out.vely.00034.pfb
new file mode 100644
index 000000000..a89b0cdf0
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00034.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00035.pfb b/test/correct_output/default_richards.out.vely.00035.pfb
new file mode 100644
index 000000000..3f02a0631
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00035.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00036.pfb b/test/correct_output/default_richards.out.vely.00036.pfb
new file mode 100644
index 000000000..fa2c27bc5
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00036.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00037.pfb b/test/correct_output/default_richards.out.vely.00037.pfb
new file mode 100644
index 000000000..ee7e92a3c
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00037.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00038.pfb b/test/correct_output/default_richards.out.vely.00038.pfb
new file mode 100644
index 000000000..d198496bd
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00038.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00039.pfb b/test/correct_output/default_richards.out.vely.00039.pfb
new file mode 100644
index 000000000..c2b30b588
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00039.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00040.pfb b/test/correct_output/default_richards.out.vely.00040.pfb
new file mode 100644
index 000000000..33229f780
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00040.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00041.pfb b/test/correct_output/default_richards.out.vely.00041.pfb
new file mode 100644
index 000000000..1ff823e38
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00041.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00042.pfb b/test/correct_output/default_richards.out.vely.00042.pfb
new file mode 100644
index 000000000..91c99e256
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00042.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00043.pfb b/test/correct_output/default_richards.out.vely.00043.pfb
new file mode 100644
index 000000000..aa3ab13a4
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00043.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00044.pfb b/test/correct_output/default_richards.out.vely.00044.pfb
new file mode 100644
index 000000000..8431971e0
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00044.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00045.pfb b/test/correct_output/default_richards.out.vely.00045.pfb
new file mode 100644
index 000000000..aae7c4226
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00045.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00046.pfb b/test/correct_output/default_richards.out.vely.00046.pfb
new file mode 100644
index 000000000..2bcab16ca
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00046.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00047.pfb b/test/correct_output/default_richards.out.vely.00047.pfb
new file mode 100644
index 000000000..240543928
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00047.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00048.pfb b/test/correct_output/default_richards.out.vely.00048.pfb
new file mode 100644
index 000000000..aa32560d0
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00048.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00049.pfb b/test/correct_output/default_richards.out.vely.00049.pfb
new file mode 100644
index 000000000..ef3b651c9
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00049.pfb differ
diff --git a/test/correct_output/default_richards.out.vely.00050.pfb b/test/correct_output/default_richards.out.vely.00050.pfb
new file mode 100644
index 000000000..d3330f1b0
Binary files /dev/null and b/test/correct_output/default_richards.out.vely.00050.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00001.pfb b/test/correct_output/default_richards.out.velz.00001.pfb
index e709a3bea..c9afbf4b2 100644
Binary files a/test/correct_output/default_richards.out.velz.00001.pfb and b/test/correct_output/default_richards.out.velz.00001.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00002.pfb b/test/correct_output/default_richards.out.velz.00002.pfb
index e0a03a2e9..ba016b213 100644
Binary files a/test/correct_output/default_richards.out.velz.00002.pfb and b/test/correct_output/default_richards.out.velz.00002.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00003.pfb b/test/correct_output/default_richards.out.velz.00003.pfb
index 4aafea23f..bbb9ad3e6 100644
Binary files a/test/correct_output/default_richards.out.velz.00003.pfb and b/test/correct_output/default_richards.out.velz.00003.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00004.pfb b/test/correct_output/default_richards.out.velz.00004.pfb
index 8f517908c..ac4426f0f 100644
Binary files a/test/correct_output/default_richards.out.velz.00004.pfb and b/test/correct_output/default_richards.out.velz.00004.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00005.pfb b/test/correct_output/default_richards.out.velz.00005.pfb
index 65f72bd95..b9ebf95cf 100644
Binary files a/test/correct_output/default_richards.out.velz.00005.pfb and b/test/correct_output/default_richards.out.velz.00005.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00006.pfb b/test/correct_output/default_richards.out.velz.00006.pfb
new file mode 100644
index 000000000..024f7fecb
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00006.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00007.pfb b/test/correct_output/default_richards.out.velz.00007.pfb
new file mode 100644
index 000000000..95fed3ad7
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00007.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00008.pfb b/test/correct_output/default_richards.out.velz.00008.pfb
new file mode 100644
index 000000000..bd9ee8c03
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00008.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00009.pfb b/test/correct_output/default_richards.out.velz.00009.pfb
new file mode 100644
index 000000000..15a06ff5d
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00009.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00010.pfb b/test/correct_output/default_richards.out.velz.00010.pfb
new file mode 100644
index 000000000..0d92184a5
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00010.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00011.pfb b/test/correct_output/default_richards.out.velz.00011.pfb
new file mode 100644
index 000000000..d60e42084
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00011.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00012.pfb b/test/correct_output/default_richards.out.velz.00012.pfb
new file mode 100644
index 000000000..6c5265eee
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00012.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00013.pfb b/test/correct_output/default_richards.out.velz.00013.pfb
new file mode 100644
index 000000000..558c0cd25
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00013.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00014.pfb b/test/correct_output/default_richards.out.velz.00014.pfb
new file mode 100644
index 000000000..f321594ad
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00014.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00015.pfb b/test/correct_output/default_richards.out.velz.00015.pfb
new file mode 100644
index 000000000..cf9f6256b
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00015.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00016.pfb b/test/correct_output/default_richards.out.velz.00016.pfb
new file mode 100644
index 000000000..945930b6f
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00016.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00017.pfb b/test/correct_output/default_richards.out.velz.00017.pfb
new file mode 100644
index 000000000..8951a2ade
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00017.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00018.pfb b/test/correct_output/default_richards.out.velz.00018.pfb
new file mode 100644
index 000000000..68a9e1060
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00018.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00019.pfb b/test/correct_output/default_richards.out.velz.00019.pfb
new file mode 100644
index 000000000..276e97b74
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00019.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00020.pfb b/test/correct_output/default_richards.out.velz.00020.pfb
new file mode 100644
index 000000000..9666333dc
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00020.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00021.pfb b/test/correct_output/default_richards.out.velz.00021.pfb
new file mode 100644
index 000000000..4735ea624
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00021.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00022.pfb b/test/correct_output/default_richards.out.velz.00022.pfb
new file mode 100644
index 000000000..c44603d4e
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00022.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00023.pfb b/test/correct_output/default_richards.out.velz.00023.pfb
new file mode 100644
index 000000000..fdfe05079
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00023.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00024.pfb b/test/correct_output/default_richards.out.velz.00024.pfb
new file mode 100644
index 000000000..452e3f9aa
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00024.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00025.pfb b/test/correct_output/default_richards.out.velz.00025.pfb
new file mode 100644
index 000000000..de8963a85
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00025.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00026.pfb b/test/correct_output/default_richards.out.velz.00026.pfb
new file mode 100644
index 000000000..54374f3c2
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00026.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00027.pfb b/test/correct_output/default_richards.out.velz.00027.pfb
new file mode 100644
index 000000000..a4f2ff1e8
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00027.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00028.pfb b/test/correct_output/default_richards.out.velz.00028.pfb
new file mode 100644
index 000000000..ff35bb752
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00028.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00029.pfb b/test/correct_output/default_richards.out.velz.00029.pfb
new file mode 100644
index 000000000..ba22f540f
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00029.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00030.pfb b/test/correct_output/default_richards.out.velz.00030.pfb
new file mode 100644
index 000000000..e7931d42d
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00030.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00031.pfb b/test/correct_output/default_richards.out.velz.00031.pfb
new file mode 100644
index 000000000..e2390a3df
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00031.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00032.pfb b/test/correct_output/default_richards.out.velz.00032.pfb
new file mode 100644
index 000000000..67ff81b34
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00032.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00033.pfb b/test/correct_output/default_richards.out.velz.00033.pfb
new file mode 100644
index 000000000..0ac17cd12
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00033.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00034.pfb b/test/correct_output/default_richards.out.velz.00034.pfb
new file mode 100644
index 000000000..ae3626085
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00034.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00035.pfb b/test/correct_output/default_richards.out.velz.00035.pfb
new file mode 100644
index 000000000..683f24cf8
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00035.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00036.pfb b/test/correct_output/default_richards.out.velz.00036.pfb
new file mode 100644
index 000000000..901d3da82
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00036.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00037.pfb b/test/correct_output/default_richards.out.velz.00037.pfb
new file mode 100644
index 000000000..b1f81d3c1
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00037.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00038.pfb b/test/correct_output/default_richards.out.velz.00038.pfb
new file mode 100644
index 000000000..f3f5336ed
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00038.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00039.pfb b/test/correct_output/default_richards.out.velz.00039.pfb
new file mode 100644
index 000000000..1f433ceee
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00039.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00040.pfb b/test/correct_output/default_richards.out.velz.00040.pfb
new file mode 100644
index 000000000..2ce65565e
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00040.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00041.pfb b/test/correct_output/default_richards.out.velz.00041.pfb
new file mode 100644
index 000000000..8fe0edbcd
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00041.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00042.pfb b/test/correct_output/default_richards.out.velz.00042.pfb
new file mode 100644
index 000000000..7b89a1215
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00042.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00043.pfb b/test/correct_output/default_richards.out.velz.00043.pfb
new file mode 100644
index 000000000..77a380ba8
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00043.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00044.pfb b/test/correct_output/default_richards.out.velz.00044.pfb
new file mode 100644
index 000000000..f18e85aa2
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00044.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00045.pfb b/test/correct_output/default_richards.out.velz.00045.pfb
new file mode 100644
index 000000000..e0693fb8d
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00045.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00046.pfb b/test/correct_output/default_richards.out.velz.00046.pfb
new file mode 100644
index 000000000..d095dd974
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00046.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00047.pfb b/test/correct_output/default_richards.out.velz.00047.pfb
new file mode 100644
index 000000000..ed8665253
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00047.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00048.pfb b/test/correct_output/default_richards.out.velz.00048.pfb
new file mode 100644
index 000000000..afea9113f
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00048.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00049.pfb b/test/correct_output/default_richards.out.velz.00049.pfb
new file mode 100644
index 000000000..d674baad4
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00049.pfb differ
diff --git a/test/correct_output/default_richards.out.velz.00050.pfb b/test/correct_output/default_richards.out.velz.00050.pfb
new file mode 100644
index 000000000..ee3216218
Binary files /dev/null and b/test/correct_output/default_richards.out.velz.00050.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00000.pfb b/test/correct_output/default_richards_restart.out.press.00000.pfb
new file mode 100644
index 000000000..549b2618a
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00000.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00001.pfb b/test/correct_output/default_richards_restart.out.press.00001.pfb
new file mode 100644
index 000000000..63235fe1c
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00001.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00002.pfb b/test/correct_output/default_richards_restart.out.press.00002.pfb
new file mode 100644
index 000000000..46409da4d
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00002.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00003.pfb b/test/correct_output/default_richards_restart.out.press.00003.pfb
new file mode 100644
index 000000000..bc023bc64
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00003.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00004.pfb b/test/correct_output/default_richards_restart.out.press.00004.pfb
new file mode 100644
index 000000000..72030f072
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00004.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00005.pfb b/test/correct_output/default_richards_restart.out.press.00005.pfb
new file mode 100644
index 000000000..926c4987c
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00005.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00006.pfb b/test/correct_output/default_richards_restart.out.press.00006.pfb
new file mode 100644
index 000000000..446e257e0
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00006.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00007.pfb b/test/correct_output/default_richards_restart.out.press.00007.pfb
new file mode 100644
index 000000000..bad289d6c
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00007.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00008.pfb b/test/correct_output/default_richards_restart.out.press.00008.pfb
new file mode 100644
index 000000000..75b90db8d
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00008.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00009.pfb b/test/correct_output/default_richards_restart.out.press.00009.pfb
new file mode 100644
index 000000000..ec1940d23
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00009.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00010.pfb b/test/correct_output/default_richards_restart.out.press.00010.pfb
new file mode 100644
index 000000000..83b63f987
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00010.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00011.pfb b/test/correct_output/default_richards_restart.out.press.00011.pfb
new file mode 100644
index 000000000..2cdd991a0
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00011.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00012.pfb b/test/correct_output/default_richards_restart.out.press.00012.pfb
new file mode 100644
index 000000000..d869dcaae
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00012.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00013.pfb b/test/correct_output/default_richards_restart.out.press.00013.pfb
new file mode 100644
index 000000000..e086dbce3
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00013.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00014.pfb b/test/correct_output/default_richards_restart.out.press.00014.pfb
new file mode 100644
index 000000000..e48878912
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00014.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00015.pfb b/test/correct_output/default_richards_restart.out.press.00015.pfb
new file mode 100644
index 000000000..a79a20bc7
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00015.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00016.pfb b/test/correct_output/default_richards_restart.out.press.00016.pfb
new file mode 100644
index 000000000..b7e976c07
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00016.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00017.pfb b/test/correct_output/default_richards_restart.out.press.00017.pfb
new file mode 100644
index 000000000..20f215a10
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00017.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00018.pfb b/test/correct_output/default_richards_restart.out.press.00018.pfb
new file mode 100644
index 000000000..be39cd26a
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00018.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00019.pfb b/test/correct_output/default_richards_restart.out.press.00019.pfb
new file mode 100644
index 000000000..a7da39d3c
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00019.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00020.pfb b/test/correct_output/default_richards_restart.out.press.00020.pfb
new file mode 100644
index 000000000..94c18fbf3
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00020.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00021.pfb b/test/correct_output/default_richards_restart.out.press.00021.pfb
new file mode 100644
index 000000000..50b058c24
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00021.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00022.pfb b/test/correct_output/default_richards_restart.out.press.00022.pfb
new file mode 100644
index 000000000..66e415bf8
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00022.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00023.pfb b/test/correct_output/default_richards_restart.out.press.00023.pfb
new file mode 100644
index 000000000..ac65e2f27
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00023.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00024.pfb b/test/correct_output/default_richards_restart.out.press.00024.pfb
new file mode 100644
index 000000000..28384ea86
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00024.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00025.pfb b/test/correct_output/default_richards_restart.out.press.00025.pfb
new file mode 100644
index 000000000..933936218
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00025.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00026.pfb b/test/correct_output/default_richards_restart.out.press.00026.pfb
new file mode 100644
index 000000000..35b4868f0
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00026.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00027.pfb b/test/correct_output/default_richards_restart.out.press.00027.pfb
new file mode 100644
index 000000000..6e32c6123
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00027.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00028.pfb b/test/correct_output/default_richards_restart.out.press.00028.pfb
new file mode 100644
index 000000000..7276f19da
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00028.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00029.pfb b/test/correct_output/default_richards_restart.out.press.00029.pfb
new file mode 100644
index 000000000..138ce7d43
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00029.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00030.pfb b/test/correct_output/default_richards_restart.out.press.00030.pfb
new file mode 100644
index 000000000..1834620e1
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00030.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00031.pfb b/test/correct_output/default_richards_restart.out.press.00031.pfb
new file mode 100644
index 000000000..ca7a39b52
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00031.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00032.pfb b/test/correct_output/default_richards_restart.out.press.00032.pfb
new file mode 100644
index 000000000..3f8a3c797
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00032.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00033.pfb b/test/correct_output/default_richards_restart.out.press.00033.pfb
new file mode 100644
index 000000000..9895b2b1a
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00033.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00034.pfb b/test/correct_output/default_richards_restart.out.press.00034.pfb
new file mode 100644
index 000000000..4b0b8dc38
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00034.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00035.pfb b/test/correct_output/default_richards_restart.out.press.00035.pfb
new file mode 100644
index 000000000..4a649282d
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00035.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00036.pfb b/test/correct_output/default_richards_restart.out.press.00036.pfb
new file mode 100644
index 000000000..548a495b5
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00036.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00037.pfb b/test/correct_output/default_richards_restart.out.press.00037.pfb
new file mode 100644
index 000000000..c4dcc8831
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00037.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00038.pfb b/test/correct_output/default_richards_restart.out.press.00038.pfb
new file mode 100644
index 000000000..101f61149
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00038.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00039.pfb b/test/correct_output/default_richards_restart.out.press.00039.pfb
new file mode 100644
index 000000000..4391793de
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00039.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00040.pfb b/test/correct_output/default_richards_restart.out.press.00040.pfb
new file mode 100644
index 000000000..72ce7badc
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00040.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00041.pfb b/test/correct_output/default_richards_restart.out.press.00041.pfb
new file mode 100644
index 000000000..a9dbef192
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00041.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00042.pfb b/test/correct_output/default_richards_restart.out.press.00042.pfb
new file mode 100644
index 000000000..fbe6251df
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00042.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00043.pfb b/test/correct_output/default_richards_restart.out.press.00043.pfb
new file mode 100644
index 000000000..b50f2b984
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00043.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00044.pfb b/test/correct_output/default_richards_restart.out.press.00044.pfb
new file mode 100644
index 000000000..fc9fec318
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00044.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00045.pfb b/test/correct_output/default_richards_restart.out.press.00045.pfb
new file mode 100644
index 000000000..ac26dce48
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00045.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00046.pfb b/test/correct_output/default_richards_restart.out.press.00046.pfb
new file mode 100644
index 000000000..9f7cdcb70
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00046.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00047.pfb b/test/correct_output/default_richards_restart.out.press.00047.pfb
new file mode 100644
index 000000000..d430f4fcf
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00047.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00048.pfb b/test/correct_output/default_richards_restart.out.press.00048.pfb
new file mode 100644
index 000000000..962762ce9
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00048.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00049.pfb b/test/correct_output/default_richards_restart.out.press.00049.pfb
new file mode 100644
index 000000000..4646a8fd0
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00049.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.press.00050.pfb b/test/correct_output/default_richards_restart.out.press.00050.pfb
new file mode 100644
index 000000000..018db8944
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.press.00050.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00000.pfb b/test/correct_output/default_richards_restart.out.satur.00000.pfb
new file mode 100644
index 000000000..1e77d92db
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00000.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00001.pfb b/test/correct_output/default_richards_restart.out.satur.00001.pfb
new file mode 100644
index 000000000..f1243f426
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00001.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00002.pfb b/test/correct_output/default_richards_restart.out.satur.00002.pfb
new file mode 100644
index 000000000..a46e5c02e
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00002.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00003.pfb b/test/correct_output/default_richards_restart.out.satur.00003.pfb
new file mode 100644
index 000000000..3910dc50e
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00003.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00004.pfb b/test/correct_output/default_richards_restart.out.satur.00004.pfb
new file mode 100644
index 000000000..9e4823fa5
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00004.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00005.pfb b/test/correct_output/default_richards_restart.out.satur.00005.pfb
new file mode 100644
index 000000000..4baab7644
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00005.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00006.pfb b/test/correct_output/default_richards_restart.out.satur.00006.pfb
new file mode 100644
index 000000000..b398007b7
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00006.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00007.pfb b/test/correct_output/default_richards_restart.out.satur.00007.pfb
new file mode 100644
index 000000000..04305a0bf
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00007.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00008.pfb b/test/correct_output/default_richards_restart.out.satur.00008.pfb
new file mode 100644
index 000000000..2be6fe392
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00008.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00009.pfb b/test/correct_output/default_richards_restart.out.satur.00009.pfb
new file mode 100644
index 000000000..bb58d67bf
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00009.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00010.pfb b/test/correct_output/default_richards_restart.out.satur.00010.pfb
new file mode 100644
index 000000000..b586d2abf
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00010.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00011.pfb b/test/correct_output/default_richards_restart.out.satur.00011.pfb
new file mode 100644
index 000000000..a39a026b8
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00011.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00012.pfb b/test/correct_output/default_richards_restart.out.satur.00012.pfb
new file mode 100644
index 000000000..e4575cc0e
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00012.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00013.pfb b/test/correct_output/default_richards_restart.out.satur.00013.pfb
new file mode 100644
index 000000000..3d5e1201d
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00013.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00014.pfb b/test/correct_output/default_richards_restart.out.satur.00014.pfb
new file mode 100644
index 000000000..df514b10e
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00014.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00015.pfb b/test/correct_output/default_richards_restart.out.satur.00015.pfb
new file mode 100644
index 000000000..e06e4fc27
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00015.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00016.pfb b/test/correct_output/default_richards_restart.out.satur.00016.pfb
new file mode 100644
index 000000000..d301ecd9a
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00016.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00017.pfb b/test/correct_output/default_richards_restart.out.satur.00017.pfb
new file mode 100644
index 000000000..395b3beef
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00017.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00018.pfb b/test/correct_output/default_richards_restart.out.satur.00018.pfb
new file mode 100644
index 000000000..fcea60a18
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00018.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00019.pfb b/test/correct_output/default_richards_restart.out.satur.00019.pfb
new file mode 100644
index 000000000..03d9f06f6
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00019.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00020.pfb b/test/correct_output/default_richards_restart.out.satur.00020.pfb
new file mode 100644
index 000000000..903e87909
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00020.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00021.pfb b/test/correct_output/default_richards_restart.out.satur.00021.pfb
new file mode 100644
index 000000000..fd3ec3ccf
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00021.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00022.pfb b/test/correct_output/default_richards_restart.out.satur.00022.pfb
new file mode 100644
index 000000000..c03e0196f
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00022.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00023.pfb b/test/correct_output/default_richards_restart.out.satur.00023.pfb
new file mode 100644
index 000000000..776878e8e
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00023.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00024.pfb b/test/correct_output/default_richards_restart.out.satur.00024.pfb
new file mode 100644
index 000000000..9a2c73b20
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00024.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00025.pfb b/test/correct_output/default_richards_restart.out.satur.00025.pfb
new file mode 100644
index 000000000..0ad5d3192
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00025.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00026.pfb b/test/correct_output/default_richards_restart.out.satur.00026.pfb
new file mode 100644
index 000000000..fd404891f
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00026.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00027.pfb b/test/correct_output/default_richards_restart.out.satur.00027.pfb
new file mode 100644
index 000000000..25f8ad267
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00027.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00028.pfb b/test/correct_output/default_richards_restart.out.satur.00028.pfb
new file mode 100644
index 000000000..ab62b4a81
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00028.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00029.pfb b/test/correct_output/default_richards_restart.out.satur.00029.pfb
new file mode 100644
index 000000000..5659eff6d
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00029.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00030.pfb b/test/correct_output/default_richards_restart.out.satur.00030.pfb
new file mode 100644
index 000000000..2d5010978
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00030.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00031.pfb b/test/correct_output/default_richards_restart.out.satur.00031.pfb
new file mode 100644
index 000000000..231d51ca2
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00031.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00032.pfb b/test/correct_output/default_richards_restart.out.satur.00032.pfb
new file mode 100644
index 000000000..84d871918
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00032.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00033.pfb b/test/correct_output/default_richards_restart.out.satur.00033.pfb
new file mode 100644
index 000000000..7eb8b7806
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00033.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00034.pfb b/test/correct_output/default_richards_restart.out.satur.00034.pfb
new file mode 100644
index 000000000..74b9b3b59
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00034.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00035.pfb b/test/correct_output/default_richards_restart.out.satur.00035.pfb
new file mode 100644
index 000000000..0efd6f096
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00035.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00036.pfb b/test/correct_output/default_richards_restart.out.satur.00036.pfb
new file mode 100644
index 000000000..04b70628a
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00036.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00037.pfb b/test/correct_output/default_richards_restart.out.satur.00037.pfb
new file mode 100644
index 000000000..77d1f055a
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00037.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00038.pfb b/test/correct_output/default_richards_restart.out.satur.00038.pfb
new file mode 100644
index 000000000..5d68db324
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00038.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00039.pfb b/test/correct_output/default_richards_restart.out.satur.00039.pfb
new file mode 100644
index 000000000..d40770bba
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00039.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00040.pfb b/test/correct_output/default_richards_restart.out.satur.00040.pfb
new file mode 100644
index 000000000..32faf889e
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00040.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00041.pfb b/test/correct_output/default_richards_restart.out.satur.00041.pfb
new file mode 100644
index 000000000..7ed88c1f8
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00041.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00042.pfb b/test/correct_output/default_richards_restart.out.satur.00042.pfb
new file mode 100644
index 000000000..8e278b9a0
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00042.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00043.pfb b/test/correct_output/default_richards_restart.out.satur.00043.pfb
new file mode 100644
index 000000000..5950ffe14
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00043.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00044.pfb b/test/correct_output/default_richards_restart.out.satur.00044.pfb
new file mode 100644
index 000000000..54d40596e
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00044.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00045.pfb b/test/correct_output/default_richards_restart.out.satur.00045.pfb
new file mode 100644
index 000000000..6a20b69f9
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00045.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00046.pfb b/test/correct_output/default_richards_restart.out.satur.00046.pfb
new file mode 100644
index 000000000..a74a5d7d6
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00046.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00047.pfb b/test/correct_output/default_richards_restart.out.satur.00047.pfb
new file mode 100644
index 000000000..d07e00e1d
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00047.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00048.pfb b/test/correct_output/default_richards_restart.out.satur.00048.pfb
new file mode 100644
index 000000000..4596a8087
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00048.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00049.pfb b/test/correct_output/default_richards_restart.out.satur.00049.pfb
new file mode 100644
index 000000000..886da7ed1
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00049.pfb differ
diff --git a/test/correct_output/default_richards_restart.out.satur.00050.pfb b/test/correct_output/default_richards_restart.out.satur.00050.pfb
new file mode 100644
index 000000000..ec7bd7863
Binary files /dev/null and b/test/correct_output/default_richards_restart.out.satur.00050.pfb differ
diff --git a/test/correct_output/indicator_field.out.perm_x.pfb b/test/correct_output/indicator_field.out.perm_x.pfb
new file mode 100644
index 000000000..307a59d32
Binary files /dev/null and b/test/correct_output/indicator_field.out.perm_x.pfb differ
diff --git a/test/correct_output/indicator_field.out.perm_y.pfb b/test/correct_output/indicator_field.out.perm_y.pfb
new file mode 100644
index 000000000..307a59d32
Binary files /dev/null and b/test/correct_output/indicator_field.out.perm_y.pfb differ
diff --git a/test/correct_output/indicator_field.out.perm_z.pfb b/test/correct_output/indicator_field.out.perm_z.pfb
new file mode 100644
index 000000000..307a59d32
Binary files /dev/null and b/test/correct_output/indicator_field.out.perm_z.pfb differ
diff --git a/test/correct_output/indicator_field.out.porosity.pfb b/test/correct_output/indicator_field.out.porosity.pfb
new file mode 100644
index 000000000..8965b3604
Binary files /dev/null and b/test/correct_output/indicator_field.out.porosity.pfb differ
diff --git a/test/correct_output/indicator_field.out.press.00000.pfb b/test/correct_output/indicator_field.out.press.00000.pfb
new file mode 100644
index 000000000..5fa4ae2e3
Binary files /dev/null and b/test/correct_output/indicator_field.out.press.00000.pfb differ
diff --git a/test/correct_output/indicator_field.out.press.00001.pfb b/test/correct_output/indicator_field.out.press.00001.pfb
new file mode 100644
index 000000000..601c62c2f
Binary files /dev/null and b/test/correct_output/indicator_field.out.press.00001.pfb differ
diff --git a/test/correct_output/indicator_field.out.satur.00000.pfb b/test/correct_output/indicator_field.out.satur.00000.pfb
new file mode 100644
index 000000000..89cd163d6
Binary files /dev/null and b/test/correct_output/indicator_field.out.satur.00000.pfb differ
diff --git a/test/correct_output/indicator_field.out.satur.00001.pfb b/test/correct_output/indicator_field.out.satur.00001.pfb
new file mode 100644
index 000000000..c34d2679c
Binary files /dev/null and b/test/correct_output/indicator_field.out.satur.00001.pfb differ
diff --git a/test/correct_output/tilted_v_with_reservoir_overland_flow.out.press.00010.pfb b/test/correct_output/tilted_v_with_reservoir_overland_flow.out.press.00010.pfb
new file mode 100644
index 000000000..afcc9717d
Binary files /dev/null and b/test/correct_output/tilted_v_with_reservoir_overland_flow.out.press.00010.pfb differ
diff --git a/test/correct_output/tilted_v_with_reservoir_overland_kinematic.out.press.00010.pfb b/test/correct_output/tilted_v_with_reservoir_overland_kinematic.out.press.00010.pfb
new file mode 100644
index 000000000..67fdb0f0d
Binary files /dev/null and b/test/correct_output/tilted_v_with_reservoir_overland_kinematic.out.press.00010.pfb differ
diff --git a/test/input/default_single_input_porosity.pfb b/test/input/default_single_input_porosity.pfb
new file mode 100644
index 000000000..b2bd7097b
Binary files /dev/null and b/test/input/default_single_input_porosity.pfb differ
diff --git a/test/input/tilted_v_with_reservoir.pfsol b/test/input/tilted_v_with_reservoir.pfsol
new file mode 100755
index 000000000..10036a239
--- /dev/null
+++ b/test/input/tilted_v_with_reservoir.pfsol
@@ -0,0 +1,2417 @@
+1
+483
+0 0 0
+1 0 0
+2 0 0
+3 0 0
+4 0 0
+5 0 0
+6 0 0
+7 0 0
+8 0 0
+9 0 0
+10 0 0
+11 0 0
+12 0 0
+13 0 0
+14 0 0
+15 0 0
+16 0 0
+17 0 0
+18 0 0
+19 0 0
+20 0 0
+21 0 0
+22 0 0
+23 0 0
+24 0 0
+25 0 0
+0 1 0
+1 1 0
+2 1 0
+3 1 0
+4 1 0
+5 1 0
+6 1 0
+7 1 0
+8 1 0
+9 1 0
+10 1 0
+11 1 0
+12 1 0
+13 1 0
+14 1 0
+15 1 0
+16 1 0
+17 1 0
+18 1 0
+19 1 0
+20 1 0
+21 1 0
+22 1 0
+23 1 0
+24 1 0
+25 1 0
+0 2 0
+1 2 0
+24 2 0
+25 2 0
+0 3 0
+1 3 0
+18 5.59375 0
+24 3 0
+25 3 0
+0 4 0
+1 4 0
+24 4 0
+25 4 0
+0 5 0
+1 5 0
+22.125 3.59375 0
+24 5 0
+25 5 0
+0 6 0
+1 6 0
+24 6 0
+25 6 0
+0 7 0
+1 7 0
+24 7 0
+25 7 0
+0 8 0
+1 8 0
+5.640625 7.2578125 0
+24 8 0
+25 8 0
+0 9 0
+1 9 0
+24 9 0
+25 9 0
+0 10 0
+1 10 0
+24 10 0
+25 10 0
+0 11 0
+1 11 0
+17.9345703125 18.220703125 0
+24 11 0
+25 11 0
+0 12 0
+1 12 0
+24 12 0
+25 12 0
+0 13 0
+1 13 0
+24 13 0
+25 13 0
+0 14 0
+1 14 0
+24 14 0
+25 14 0
+0 15 0
+1 15 0
+24 15 0
+25 15 0
+0 16 0
+1 16 0
+24 16 0
+25 16 0
+0 17 0
+1 17 0
+24 17 0
+25 17 0
+0 18 0
+1 18 0
+24 18 0
+25 18 0
+0 19 0
+1 19 0
+24 19 0
+25 19 0
+0 20 0
+1 20 0
+5.47265625 16.98046875 0
+24 20 0
+25 20 0
+0 21 0
+1 21 0
+3.15625 21.75 0
+24 21 0
+25 21 0
+0 22 0
+1 22 0
+8.625 22.25 0
+24 22 0
+25 22 0
+0 23 0
+1 23 0
+24 23 0
+25 23 0
+0 24 0
+1 24 0
+2 24 0
+3 24 0
+4 24 0
+5 24 0
+6 24 0
+7 24 0
+8 24 0
+9 24 0
+10 24 0
+11 24 0
+12 24 0
+13 24 0
+14 24 0
+15 24 0
+16 24 0
+17 24 0
+18 24 0
+19 24 0
+20 24 0
+21 24 0
+22 24 0
+23 24 0
+24 24 0
+25 24 0
+0 25 0
+1 25 0
+2 25 0
+3 25 0
+4 25 0
+5 25 0
+6 25 0
+7 25 0
+8 25 0
+9 25 0
+10 25 0
+11 25 0
+12 25 0
+13 25 0
+14 25 0
+15 25 0
+16 25 0
+17 25 0
+18 25 0
+19 25 0
+20 25 0
+21 25 0
+22 25 0
+23 25 0
+24 25 0
+25 25 0
+0 0 0.5
+1 0 0.5
+2 0 0.5
+3 0 0.5
+4 0 0.5
+5 0 0.5
+6 0 0.5
+7 0 0.5
+8 0 0.5
+9 0 0.5
+10 0 0.5
+11 0 0.5
+12 0 0.5
+13 0 0.5
+14 0 0.5
+15 0 0.5
+16 0 0.5
+17 0 0.5
+18 0 0.5
+19 0 0.5
+20 0 0.5
+21 0 0.5
+22 0 0.5
+23 0 0.5
+24 0 0.5
+25 0 0.5
+0 1 0.5
+1 1 0.5
+2 1 0.5
+3 1 0.5
+4 1 0.5
+5 1 0.5
+6 1 0.5
+7 1 0.5
+8 1 0.5
+9 1 0.5
+10 1 0.5
+11 1 0.5
+12 1 0.5
+13 1 0.5
+14 1 0.5
+15 1 0.5
+16 1 0.5
+17 1 0.5
+18 1 0.5
+19 1 0.5
+20 1 0.5
+21 1 0.5
+22 1 0.5
+23 1 0.5
+24 1 0.5
+25 1 0.5
+0 2 0.5
+1 2 0.5
+24 2 0.5
+25 2 0.5
+0 3 0.5
+1 3 0.5
+24 3 0.5
+25 3 0.5
+0 4 0.5
+1 4 0.5
+24 4 0.5
+25 4 0.5
+0 5 0.5
+1 5 0.5
+24 5 0.5
+25 5 0.5
+0 6 0.5
+1 6 0.5
+7.9765625 6.8515625 0.5
+24 6 0.5
+25 6 0.5
+0 7 0.5
+1 7 0.5
+24 7 0.5
+25 7 0.5
+0 8 0.5
+1 8 0.5
+24 8 0.5
+25 8 0.5
+0 9 0.5
+1 9 0.5
+24 9 0.5
+25 9 0.5
+0 10 0.5
+1 10 0.5
+16.6171875 7.7265625 0.5
+24 10 0.5
+25 10 0.5
+0 11 0.5
+1 11 0.5
+2 11 0.5
+3 11 0.5
+4 11 0.5
+5 11 0.5
+6 11 0.5
+7 11 0.5
+8 11 0.5
+9 11 0.5
+10 11 0.5
+11 11 0.5
+12 11 0.5
+13 11 0.5
+14 11 0.5
+15 11 0.5
+16 11 0.5
+17 11 0.5
+18 11 0.5
+19 11 0.5
+20 11 0.5
+21 11 0.5
+22 11 0.5
+23 11 0.5
+24 11 0.5
+25 11 0.5
+0 12 0.5
+1 12 0.5
+2 12 0.5
+3 12 0.5
+4 12 0.5
+5 12 0.5
+6 12 0.5
+7 12 0.5
+8 12 0.5
+9 12 0.5
+10 12 0.5
+11 12 0.5
+12 12 0.5
+13 12 0.5
+14 12 0.5
+15 12 0.5
+16 12 0.5
+17 12 0.5
+18 12 0.5
+19 12 0.5
+20 12 0.5
+21 12 0.5
+22 12 0.5
+23 12 0.5
+24 12 0.5
+25 12 0.5
+0 13 0.5
+1 13 0.5
+2 13 0.5
+3 13 0.5
+4 13 0.5
+5 13 0.5
+6 13 0.5
+7 13 0.5
+8 13 0.5
+9 13 0.5
+10 13 0.5
+11 13 0.5
+12 13 0.5
+13 13 0.5
+14 13 0.5
+15 13 0.5
+16 13 0.5
+17 13 0.5
+18 13 0.5
+19 13 0.5
+20 13 0.5
+21 13 0.5
+22 13 0.5
+23 13 0.5
+24 13 0.5
+25 13 0.5
+0 14 0.5
+1 14 0.5
+2 14 0.5
+3 14 0.5
+4 14 0.5
+5 14 0.5
+6 14 0.5
+7 14 0.5
+8 14 0.5
+9 14 0.5
+10 14 0.5
+11 14 0.5
+12 14 0.5
+13 14 0.5
+14 14 0.5
+15 14 0.5
+16 14 0.5
+17 14 0.5
+18 14 0.5
+19 14 0.5
+20 14 0.5
+21 14 0.5
+22 14 0.5
+23 14 0.5
+24 14 0.5
+25 14 0.5
+0 15 0.5
+1 15 0.5
+24 15 0.5
+25 15 0.5
+0 16 0.5
+1 16 0.5
+24 16 0.5
+25 16 0.5
+0 17 0.5
+1 17 0.5
+24 17 0.5
+25 17 0.5
+0 18 0.5
+1 18 0.5
+24 18 0.5
+25 18 0.5
+0 19 0.5
+1 19 0.5
+7.9765625 19.8515625 0.5
+24 19 0.5
+25 19 0.5
+0 20 0.5
+1 20 0.5
+24 20 0.5
+25 20 0.5
+0 21 0.5
+1 21 0.5
+24 21 0.5
+25 21 0.5
+0 22 0.5
+1 22 0.5
+24 22 0.5
+25 22 0.5
+0 23 0.5
+1 23 0.5
+16.6171875 20.7265625 0.5
+24 23 0.5
+25 23 0.5
+0 24 0.5
+1 24 0.5
+2 24 0.5
+3 24 0.5
+4 24 0.5
+5 24 0.5
+6 24 0.5
+7 24 0.5
+8 24 0.5
+9 24 0.5
+10 24 0.5
+11 24 0.5
+12 24 0.5
+13 24 0.5
+14 24 0.5
+15 24 0.5
+16 24 0.5
+17 24 0.5
+18 24 0.5
+19 24 0.5
+20 24 0.5
+21 24 0.5
+22 24 0.5
+23 24 0.5
+24 24 0.5
+25 24 0.5
+0 25 0.5
+1 25 0.5
+2 25 0.5
+3 25 0.5
+4 25 0.5
+5 25 0.5
+6 25 0.5
+7 25 0.5
+8 25 0.5
+9 25 0.5
+10 25 0.5
+11 25 0.5
+12 25 0.5
+13 25 0.5
+14 25 0.5
+15 25 0.5
+16 25 0.5
+17 25 0.5
+18 25 0.5
+19 25 0.5
+20 25 0.5
+21 25 0.5
+22 25 0.5
+23 25 0.5
+24 25 0.5
+25 25 0.5
+1
+962
+199 200 226
+199 226 225
+0 27 1
+0 26 27
+0 199 26
+199 225 26
+0 1 200
+0 200 199
+200 201 227
+200 227 226
+1 28 2
+1 27 28
+1 2 201
+1 201 200
+201 202 228
+201 228 227
+2 29 3
+2 28 29
+2 3 202
+2 202 201
+202 203 229
+202 229 228
+3 30 4
+3 29 30
+3 4 203
+3 203 202
+203 204 230
+203 230 229
+4 31 5
+4 30 31
+4 5 204
+4 204 203
+204 205 231
+204 231 230
+5 32 6
+5 31 32
+5 6 205
+5 205 204
+205 206 232
+205 232 231
+6 33 7
+6 32 33
+6 7 206
+6 206 205
+206 207 233
+206 233 232
+7 34 8
+7 33 34
+7 8 207
+7 207 206
+207 208 234
+207 234 233
+8 35 9
+8 34 35
+8 9 208
+8 208 207
+208 209 235
+208 235 234
+9 36 10
+9 35 36
+9 10 209
+9 209 208
+209 210 236
+209 236 235
+10 37 11
+10 36 37
+10 11 210
+10 210 209
+210 211 237
+210 237 236
+11 38 12
+11 37 38
+11 12 211
+11 211 210
+211 212 238
+211 238 237
+12 39 13
+12 38 39
+12 13 212
+12 212 211
+212 213 239
+212 239 238
+13 40 14
+13 39 40
+13 14 213
+13 213 212
+213 214 240
+213 240 239
+14 41 15
+14 40 41
+14 15 214
+14 214 213
+214 215 241
+214 241 240
+15 42 16
+15 41 42
+15 16 215
+15 215 214
+215 216 242
+215 242 241
+16 43 17
+16 42 43
+16 17 216
+16 216 215
+216 217 243
+216 243 242
+17 44 18
+17 43 44
+17 18 217
+17 217 216
+217 218 244
+217 244 243
+18 45 19
+18 44 45
+18 19 218
+18 218 217
+218 219 245
+218 245 244
+19 46 20
+19 45 46
+19 20 219
+19 219 218
+219 220 246
+219 246 245
+20 47 21
+20 46 47
+20 21 220
+20 220 219
+220 221 247
+220 247 246
+21 48 22
+21 47 48
+21 22 221
+21 221 220
+221 222 248
+221 248 247
+22 49 23
+22 48 49
+22 23 222
+22 222 221
+222 223 249
+222 249 248
+23 50 24
+23 49 50
+23 24 223
+23 223 222
+223 224 250
+223 250 249
+24 51 25
+24 50 51
+51 224 25
+224 51 250
+24 25 224
+24 224 223
+225 226 252
+225 252 251
+26 53 27
+26 52 53
+26 225 52
+225 251 52
+226 227 269
+226 269 252
+27 80 28
+27 53 80
+227 228 269
+28 80 29
+228 229 269
+29 80 30
+229 230 269
+30 80 31
+230 231 269
+31 80 32
+231 232 269
+32 80 33
+232 233 269
+33 80 34
+233 234 269
+34 80 35
+234 235 286
+234 286 269
+35 58 36
+35 80 58
+235 236 286
+36 58 37
+236 237 286
+37 58 38
+237 238 286
+38 58 39
+238 239 286
+39 58 40
+239 240 286
+40 58 41
+240 241 286
+41 58 42
+241 242 286
+42 58 43
+242 243 286
+43 67 44
+43 58 67
+243 244 286
+44 67 45
+244 245 286
+45 67 46
+245 246 286
+46 67 47
+246 247 286
+47 67 48
+247 248 286
+48 67 49
+248 249 253
+248 253 286
+49 54 50
+49 67 54
+249 250 254
+249 254 253
+50 55 51
+50 54 55
+55 250 51
+250 55 254
+251 252 256
+251 256 255
+52 57 53
+52 56 57
+52 251 56
+251 255 56
+252 269 256
+53 57 80
+80 93 58
+286 253 257
+67 59 54
+253 254 258
+253 258 257
+54 60 55
+54 59 60
+60 254 55
+254 60 258
+255 256 260
+255 260 259
+56 62 57
+56 61 62
+56 255 61
+255 259 61
+256 269 260
+57 62 80
+286 257 261
+67 63 59
+257 258 262
+257 262 261
+59 64 60
+59 63 64
+64 258 60
+258 64 262
+259 260 264
+259 264 263
+61 66 62
+61 65 66
+61 259 65
+259 263 65
+260 269 264
+62 66 80
+286 261 265
+67 68 63
+261 262 266
+261 266 265
+63 69 64
+63 68 69
+69 262 64
+262 69 266
+263 264 268
+263 268 267
+65 71 66
+65 70 71
+65 263 70
+263 267 70
+264 269 268
+66 71 80
+286 265 270
+67 72 68
+265 266 271
+265 271 270
+68 73 69
+68 72 73
+73 266 69
+266 73 271
+267 268 273
+267 273 272
+70 75 71
+70 74 75
+70 267 74
+267 272 74
+268 269 273
+71 75 80
+286 270 274
+67 76 72
+67 58 76
+270 271 275
+270 275 274
+72 77 73
+72 76 77
+77 271 73
+271 77 275
+272 273 277
+272 277 276
+74 79 75
+74 78 79
+74 272 78
+272 276 78
+273 269 277
+75 79 80
+286 274 278
+58 81 76
+274 275 279
+274 279 278
+76 82 77
+76 81 82
+82 275 77
+275 82 279
+276 277 281
+276 281 280
+78 84 79
+78 83 84
+78 276 83
+276 280 83
+277 269 281
+79 130 80
+79 84 130
+286 278 282
+58 85 81
+278 279 283
+278 283 282
+81 86 82
+81 85 86
+86 279 82
+279 86 283
+280 281 285
+280 285 284
+83 88 84
+83 87 88
+83 280 87
+280 284 87
+281 269 285
+84 88 130
+286 282 287
+58 89 85
+282 283 288
+282 288 287
+85 90 86
+85 89 90
+90 283 86
+283 90 288
+284 285 290
+284 290 289
+87 92 88
+87 91 92
+87 284 91
+284 289 91
+285 269 291
+285 291 290
+88 92 130
+269 292 291
+269 293 292
+269 294 293
+269 295 294
+269 296 295
+269 297 296
+269 298 297
+269 299 298
+269 300 299
+269 301 300
+269 302 301
+269 303 302
+269 286 304
+269 304 303
+286 305 304
+286 306 305
+286 307 306
+286 308 307
+286 309 308
+286 310 309
+286 311 310
+286 312 311
+286 287 313
+286 313 312
+58 94 89
+58 93 94
+287 288 314
+287 314 313
+89 95 90
+89 94 95
+95 288 90
+288 95 314
+289 290 316
+289 316 315
+91 97 92
+91 96 97
+91 289 96
+289 315 96
+290 291 317
+290 317 316
+92 97 130
+291 292 318
+291 318 317
+292 293 319
+292 319 318
+293 294 320
+293 320 319
+294 295 321
+294 321 320
+295 296 322
+295 322 321
+296 297 323
+296 323 322
+297 298 324
+297 324 323
+298 299 325
+298 325 324
+299 300 326
+299 326 325
+300 301 327
+300 327 326
+301 302 328
+301 328 327
+302 303 329
+302 329 328
+303 304 330
+303 330 329
+304 305 331
+304 331 330
+305 306 332
+305 332 331
+306 307 333
+306 333 332
+307 308 334
+307 334 333
+308 309 335
+308 335 334
+309 310 336
+309 336 335
+310 311 337
+310 337 336
+311 312 338
+311 338 337
+312 313 339
+312 339 338
+93 98 94
+313 314 340
+313 340 339
+94 99 95
+94 98 99
+99 314 95
+314 99 340
+315 316 342
+315 342 341
+96 101 97
+96 100 101
+96 315 100
+315 341 100
+316 317 343
+316 343 342
+97 101 130
+317 318 344
+317 344 343
+318 319 345
+318 345 344
+319 320 346
+319 346 345
+320 321 347
+320 347 346
+321 322 348
+321 348 347
+322 323 349
+322 349 348
+323 324 350
+323 350 349
+324 325 351
+324 351 350
+325 326 352
+325 352 351
+326 327 353
+326 353 352
+327 328 354
+327 354 353
+328 329 355
+328 355 354
+329 330 356
+329 356 355
+330 331 357
+330 357 356
+331 332 358
+331 358 357
+332 333 359
+332 359 358
+333 334 360
+333 360 359
+334 335 361
+334 361 360
+335 336 362
+335 362 361
+336 337 363
+336 363 362
+337 338 364
+337 364 363
+338 339 365
+338 365 364
+93 102 98
+339 340 366
+339 366 365
+98 103 99
+98 102 103
+103 340 99
+340 103 366
+341 342 368
+341 368 367
+100 105 101
+100 104 105
+100 341 104
+341 367 104
+342 343 369
+342 369 368
+101 105 130
+343 344 370
+343 370 369
+344 345 371
+344 371 370
+345 346 372
+345 372 371
+346 347 373
+346 373 372
+347 348 374
+347 374 373
+348 349 375
+348 375 374
+349 350 376
+349 376 375
+350 351 377
+350 377 376
+351 352 378
+351 378 377
+352 353 379
+352 379 378
+353 354 380
+353 380 379
+354 355 381
+354 381 380
+355 356 382
+355 382 381
+356 357 383
+356 383 382
+357 358 384
+357 384 383
+358 359 385
+358 385 384
+359 360 386
+359 386 385
+360 361 387
+360 387 386
+361 362 388
+361 388 387
+362 363 389
+362 389 388
+363 364 390
+363 390 389
+364 365 391
+364 391 390
+93 106 102
+365 366 392
+365 392 391
+102 107 103
+102 106 107
+107 366 103
+366 107 392
+367 368 394
+367 394 393
+104 109 105
+104 108 109
+104 367 108
+367 393 108
+368 369 411
+368 411 394
+105 109 130
+369 370 411
+370 371 411
+371 372 411
+372 373 411
+373 374 411
+374 375 411
+375 376 411
+376 377 428
+376 428 411
+377 378 428
+378 379 428
+379 380 428
+380 381 428
+381 382 428
+382 383 428
+383 384 428
+384 385 428
+385 386 428
+386 387 428
+387 388 428
+388 389 428
+389 390 428
+390 391 395
+390 395 428
+93 110 106
+391 392 396
+391 396 395
+106 111 107
+106 110 111
+111 392 107
+392 111 396
+393 394 398
+393 398 397
+108 113 109
+108 112 113
+108 393 112
+393 397 112
+394 411 398
+109 113 130
+428 395 399
+93 114 110
+395 396 400
+395 400 399
+110 115 111
+110 114 115
+115 396 111
+396 115 400
+397 398 402
+397 402 401
+112 117 113
+112 116 117
+112 397 116
+397 401 116
+398 411 402
+113 117 130
+80 130 93
+428 399 403
+93 118 114
+399 400 404
+399 404 403
+114 119 115
+114 118 119
+119 400 115
+400 119 404
+401 402 406
+401 406 405
+116 121 117
+116 120 121
+116 401 120
+401 405 120
+402 411 406
+117 121 130
+428 403 407
+93 122 118
+403 404 408
+403 408 407
+118 123 119
+118 122 123
+123 404 119
+404 123 408
+405 406 410
+405 410 409
+120 125 121
+120 124 125
+120 405 124
+405 409 124
+406 411 410
+121 125 130
+428 407 412
+93 126 122
+407 408 413
+407 413 412
+122 127 123
+122 126 127
+127 408 123
+408 127 413
+409 410 415
+409 415 414
+124 129 125
+124 128 129
+124 409 128
+409 414 128
+410 411 415
+125 129 130
+428 412 416
+93 131 126
+412 413 417
+412 417 416
+126 132 127
+126 131 132
+132 413 127
+413 132 417
+414 415 419
+414 419 418
+128 134 129
+128 133 134
+128 414 133
+414 418 133
+415 411 419
+129 135 130
+129 134 135
+428 416 420
+93 136 131
+416 417 421
+416 421 420
+131 137 132
+131 136 137
+137 417 132
+417 137 421
+418 419 423
+418 423 422
+133 139 134
+133 138 139
+133 418 138
+418 422 138
+419 411 423
+134 139 135
+135 140 130
+130 140 93
+428 420 424
+93 141 136
+420 421 425
+420 425 424
+136 142 137
+136 141 142
+142 421 137
+421 142 425
+422 423 427
+422 427 426
+138 144 139
+138 143 144
+138 422 143
+422 426 143
+423 411 427
+139 144 135
+428 424 429
+93 145 141
+424 425 430
+424 430 429
+141 146 142
+141 145 146
+146 425 142
+425 146 430
+426 427 432
+426 432 431
+143 148 144
+143 147 148
+143 426 147
+426 431 147
+427 411 433
+427 433 432
+144 149 135
+144 148 149
+411 434 433
+135 149 150
+411 435 434
+135 150 151
+411 436 435
+135 151 152
+411 437 436
+135 152 153
+411 438 437
+135 154 140
+135 153 154
+411 439 438
+140 154 155
+411 440 439
+140 155 156
+411 441 440
+140 156 157
+411 442 441
+140 157 158
+411 443 442
+140 158 159
+411 444 443
+140 160 93
+140 159 160
+411 445 444
+93 160 161
+411 428 446
+411 446 445
+93 161 162
+428 447 446
+93 162 163
+428 448 447
+93 163 164
+428 449 448
+93 164 165
+428 450 449
+93 165 166
+428 451 450
+93 166 167
+428 452 451
+93 167 168
+428 453 452
+93 168 169
+428 454 453
+93 169 170
+428 429 455
+428 455 454
+93 171 145
+93 170 171
+429 430 456
+429 456 455
+145 172 146
+145 171 172
+172 430 146
+430 172 456
+431 432 458
+431 458 457
+147 174 148
+147 173 174
+147 431 173
+431 457 173
+458 174 173
+457 458 173
+432 433 459
+432 459 458
+148 175 149
+148 174 175
+459 175 174
+458 459 174
+433 434 460
+433 460 459
+149 176 150
+149 175 176
+460 176 175
+459 460 175
+434 435 461
+434 461 460
+150 177 151
+150 176 177
+461 177 176
+460 461 176
+435 436 462
+435 462 461
+151 178 152
+151 177 178
+462 178 177
+461 462 177
+436 437 463
+436 463 462
+152 179 153
+152 178 179
+463 179 178
+462 463 178
+437 438 464
+437 464 463
+153 180 154
+153 179 180
+464 180 179
+463 464 179
+438 439 465
+438 465 464
+154 181 155
+154 180 181
+465 181 180
+464 465 180
+439 440 466
+439 466 465
+155 182 156
+155 181 182
+466 182 181
+465 466 181
+440 441 467
+440 467 466
+156 183 157
+156 182 183
+467 183 182
+466 467 182
+441 442 468
+441 468 467
+157 184 158
+157 183 184
+468 184 183
+467 468 183
+442 443 469
+442 469 468
+158 185 159
+158 184 185
+469 185 184
+468 469 184
+443 444 470
+443 470 469
+159 186 160
+159 185 186
+470 186 185
+469 470 185
+444 445 471
+444 471 470
+160 187 161
+160 186 187
+471 187 186
+470 471 186
+445 446 472
+445 472 471
+161 188 162
+161 187 188
+472 188 187
+471 472 187
+446 447 473
+446 473 472
+162 189 163
+162 188 189
+473 189 188
+472 473 188
+447 448 474
+447 474 473
+163 190 164
+163 189 190
+474 190 189
+473 474 189
+448 449 475
+448 475 474
+164 191 165
+164 190 191
+475 191 190
+474 475 190
+449 450 476
+449 476 475
+165 192 166
+165 191 192
+476 192 191
+475 476 191
+450 451 477
+450 477 476
+166 193 167
+166 192 193
+477 193 192
+476 477 192
+451 452 478
+451 478 477
+167 194 168
+167 193 194
+478 194 193
+477 478 193
+452 453 479
+452 479 478
+168 195 169
+168 194 195
+479 195 194
+478 479 194
+453 454 480
+453 480 479
+169 196 170
+169 195 196
+480 196 195
+479 480 195
+454 455 481
+454 481 480
+170 197 171
+170 196 197
+481 197 196
+480 481 196
+455 456 482
+455 482 481
+171 198 172
+171 197 198
+198 456 172
+456 198 482
+482 198 197
+481 482 197
+5
+296
+2
+3
+10
+11
+16
+17
+22
+23
+28
+29
+34
+35
+40
+41
+46
+47
+52
+53
+58
+59
+64
+65
+70
+71
+76
+77
+82
+83
+88
+89
+94
+95
+100
+101
+106
+107
+112
+113
+118
+119
+124
+125
+130
+131
+136
+137
+142
+143
+148
+149
+156
+157
+162
+163
+165
+167
+169
+171
+173
+175
+177
+180
+181
+183
+185
+187
+189
+191
+193
+195
+197
+198
+200
+202
+204
+206
+208
+211
+212
+215
+216
+221
+222
+226
+227
+229
+232
+233
+238
+239
+243
+245
+248
+249
+254
+255
+259
+261
+264
+265
+270
+271
+275
+277
+280
+281
+286
+287
+291
+293
+294
+297
+298
+303
+304
+308
+310
+313
+314
+319
+320
+324
+325
+327
+330
+331
+336
+337
+341
+343
+346
+347
+352
+353
+358
+383
+384
+387
+388
+393
+394
+399
+444
+447
+448
+453
+454
+459
+504
+507
+508
+513
+514
+519
+564
+567
+568
+573
+574
+579
+604
+607
+608
+613
+614
+618
+620
+623
+624
+629
+630
+634
+635
+637
+640
+641
+646
+647
+651
+653
+656
+657
+662
+663
+667
+669
+672
+673
+678
+679
+683
+685
+688
+689
+694
+695
+699
+700
+702
+705
+706
+711
+712
+716
+717
+718
+720
+723
+724
+729
+730
+734
+736
+739
+740
+745
+746
+751
+752
+754
+756
+758
+760
+762
+763
+765
+767
+769
+771
+773
+775
+776
+778
+781
+783
+785
+787
+789
+791
+793
+795
+797
+800
+801
+804
+805
+810
+811
+818
+819
+824
+825
+830
+831
+836
+837
+842
+843
+848
+849
+854
+855
+860
+861
+866
+867
+872
+873
+878
+879
+884
+885
+890
+891
+896
+897
+902
+903
+908
+909
+914
+915
+920
+921
+926
+927
+932
+933
+938
+939
+944
+945
+950
+951
+956
+957
+200
+4
+5
+6
+7
+12
+13
+18
+19
+24
+25
+30
+31
+36
+37
+42
+43
+48
+49
+54
+55
+60
+61
+66
+67
+72
+73
+78
+79
+84
+85
+90
+91
+96
+97
+102
+103
+108
+109
+114
+115
+120
+121
+126
+127
+132
+133
+138
+139
+144
+145
+150
+151
+152
+153
+158
+159
+217
+218
+223
+224
+234
+235
+240
+241
+250
+251
+256
+257
+266
+267
+272
+273
+282
+283
+288
+289
+299
+300
+305
+306
+315
+316
+321
+322
+332
+333
+338
+339
+348
+349
+354
+355
+389
+390
+395
+396
+449
+450
+455
+456
+509
+510
+515
+516
+569
+570
+575
+576
+609
+610
+615
+616
+625
+626
+631
+632
+642
+643
+648
+649
+658
+659
+664
+665
+674
+675
+680
+681
+690
+691
+696
+697
+707
+708
+713
+714
+725
+726
+731
+732
+741
+742
+747
+748
+806
+807
+812
+813
+814
+815
+820
+821
+826
+827
+832
+833
+838
+839
+844
+845
+850
+851
+856
+857
+862
+863
+868
+869
+874
+875
+880
+881
+886
+887
+892
+893
+898
+899
+904
+905
+910
+911
+916
+917
+922
+923
+928
+929
+934
+935
+940
+941
+946
+947
+952
+953
+958
+959
+960
+961
+208
+511
+512
+517
+518
+520
+521
+522
+523
+524
+525
+526
+527
+528
+529
+530
+531
+532
+533
+534
+535
+536
+537
+538
+539
+540
+541
+542
+543
+544
+545
+546
+547
+548
+549
+550
+551
+552
+553
+554
+555
+556
+557
+558
+559
+560
+561
+562
+563
+565
+566
+571
+572
+577
+578
+580
+581
+582
+583
+584
+585
+586
+587
+588
+589
+590
+591
+592
+593
+594
+595
+596
+597
+598
+599
+600
+601
+602
+603
+605
+606
+611
+612
+617
+619
+621
+622
+627
+628
+633
+636
+638
+639
+644
+645
+650
+652
+654
+655
+660
+661
+666
+668
+670
+671
+676
+677
+682
+684
+686
+687
+692
+693
+698
+701
+703
+704
+709
+710
+715
+719
+721
+722
+727
+728
+733
+735
+737
+738
+743
+744
+749
+750
+753
+755
+757
+759
+761
+764
+766
+768
+770
+772
+774
+777
+779
+780
+782
+784
+786
+788
+790
+792
+794
+796
+798
+799
+802
+803
+808
+809
+816
+817
+822
+823
+828
+829
+834
+835
+840
+841
+846
+847
+852
+853
+858
+859
+864
+865
+870
+871
+876
+877
+882
+883
+888
+889
+894
+895
+900
+901
+906
+907
+912
+913
+918
+919
+924
+925
+930
+931
+936
+937
+942
+943
+948
+949
+954
+955
+50
+451
+452
+457
+458
+460
+461
+462
+463
+464
+465
+466
+467
+468
+469
+470
+471
+472
+473
+474
+475
+476
+477
+478
+479
+480
+481
+482
+483
+484
+485
+486
+487
+488
+489
+490
+491
+492
+493
+494
+495
+496
+497
+498
+499
+500
+501
+502
+503
+505
+506
+208
+0
+1
+8
+9
+14
+15
+20
+21
+26
+27
+32
+33
+38
+39
+44
+45
+50
+51
+56
+57
+62
+63
+68
+69
+74
+75
+80
+81
+86
+87
+92
+93
+98
+99
+104
+105
+110
+111
+116
+117
+122
+123
+128
+129
+134
+135
+140
+141
+146
+147
+154
+155
+160
+161
+164
+166
+168
+170
+172
+174
+176
+178
+179
+182
+184
+186
+188
+190
+192
+194
+196
+199
+201
+203
+205
+207
+209
+210
+213
+214
+219
+220
+225
+228
+230
+231
+236
+237
+242
+244
+246
+247
+252
+253
+258
+260
+262
+263
+268
+269
+274
+276
+278
+279
+284
+285
+290
+292
+295
+296
+301
+302
+307
+309
+311
+312
+317
+318
+323
+326
+328
+329
+334
+335
+340
+342
+344
+345
+350
+351
+356
+357
+359
+360
+361
+362
+363
+364
+365
+366
+367
+368
+369
+370
+371
+372
+373
+374
+375
+376
+377
+378
+379
+380
+381
+382
+385
+386
+391
+392
+397
+398
+400
+401
+402
+403
+404
+405
+406
+407
+408
+409
+410
+411
+412
+413
+414
+415
+416
+417
+418
+419
+420
+421
+422
+423
+424
+425
+426
+427
+428
+429
+430
+431
+432
+433
+434
+435
+436
+437
+438
+439
+440
+441
+442
+443
+445
+446
diff --git a/test/input/tilted_v_with_reservoir.vtk b/test/input/tilted_v_with_reservoir.vtk
new file mode 100755
index 000000000..1e361bfde
--- /dev/null
+++ b/test/input/tilted_v_with_reservoir.vtk
@@ -0,0 +1,2416 @@
+# vtk DataFile Version 2.0
+./Xriver_nx25_dz0.5.vtk
+ASCII
+DATASET POLYDATA
+POINTS 483 float
+0 0 0
+1 0 0
+2 0 0
+3 0 0
+4 0 0
+5 0 0
+6 0 0
+7 0 0
+8 0 0
+9 0 0
+10 0 0
+11 0 0
+12 0 0
+13 0 0
+14 0 0
+15 0 0
+16 0 0
+17 0 0
+18 0 0
+19 0 0
+20 0 0
+21 0 0
+22 0 0
+23 0 0
+24 0 0
+25 0 0
+0 1 0
+1 1 0
+2 1 0
+3 1 0
+4 1 0
+5 1 0
+6 1 0
+7 1 0
+8 1 0
+9 1 0
+10 1 0
+11 1 0
+12 1 0
+13 1 0
+14 1 0
+15 1 0
+16 1 0
+17 1 0
+18 1 0
+19 1 0
+20 1 0
+21 1 0
+22 1 0
+23 1 0
+24 1 0
+25 1 0
+0 2 0
+1 2 0
+24 2 0
+25 2 0
+0 3 0
+1 3 0
+18 5.59375 0
+24 3 0
+25 3 0
+0 4 0
+1 4 0
+24 4 0
+25 4 0
+0 5 0
+1 5 0
+22.125 3.59375 0
+24 5 0
+25 5 0
+0 6 0
+1 6 0
+24 6 0
+25 6 0
+0 7 0
+1 7 0
+24 7 0
+25 7 0
+0 8 0
+1 8 0
+5.640625 7.2578125 0
+24 8 0
+25 8 0
+0 9 0
+1 9 0
+24 9 0
+25 9 0
+0 10 0
+1 10 0
+24 10 0
+25 10 0
+0 11 0
+1 11 0
+17.9345703125 18.220703125 0
+24 11 0
+25 11 0
+0 12 0
+1 12 0
+24 12 0
+25 12 0
+0 13 0
+1 13 0
+24 13 0
+25 13 0
+0 14 0
+1 14 0
+24 14 0
+25 14 0
+0 15 0
+1 15 0
+24 15 0
+25 15 0
+0 16 0
+1 16 0
+24 16 0
+25 16 0
+0 17 0
+1 17 0
+24 17 0
+25 17 0
+0 18 0
+1 18 0
+24 18 0
+25 18 0
+0 19 0
+1 19 0
+24 19 0
+25 19 0
+0 20 0
+1 20 0
+5.47265625 16.98046875 0
+24 20 0
+25 20 0
+0 21 0
+1 21 0
+3.15625 21.75 0
+24 21 0
+25 21 0
+0 22 0
+1 22 0
+8.625 22.25 0
+24 22 0
+25 22 0
+0 23 0
+1 23 0
+24 23 0
+25 23 0
+0 24 0
+1 24 0
+2 24 0
+3 24 0
+4 24 0
+5 24 0
+6 24 0
+7 24 0
+8 24 0
+9 24 0
+10 24 0
+11 24 0
+12 24 0
+13 24 0
+14 24 0
+15 24 0
+16 24 0
+17 24 0
+18 24 0
+19 24 0
+20 24 0
+21 24 0
+22 24 0
+23 24 0
+24 24 0
+25 24 0
+0 25 0
+1 25 0
+2 25 0
+3 25 0
+4 25 0
+5 25 0
+6 25 0
+7 25 0
+8 25 0
+9 25 0
+10 25 0
+11 25 0
+12 25 0
+13 25 0
+14 25 0
+15 25 0
+16 25 0
+17 25 0
+18 25 0
+19 25 0
+20 25 0
+21 25 0
+22 25 0
+23 25 0
+24 25 0
+25 25 0
+0 0 0.5
+1 0 0.5
+2 0 0.5
+3 0 0.5
+4 0 0.5
+5 0 0.5
+6 0 0.5
+7 0 0.5
+8 0 0.5
+9 0 0.5
+10 0 0.5
+11 0 0.5
+12 0 0.5
+13 0 0.5
+14 0 0.5
+15 0 0.5
+16 0 0.5
+17 0 0.5
+18 0 0.5
+19 0 0.5
+20 0 0.5
+21 0 0.5
+22 0 0.5
+23 0 0.5
+24 0 0.5
+25 0 0.5
+0 1 0.5
+1 1 0.5
+2 1 0.5
+3 1 0.5
+4 1 0.5
+5 1 0.5
+6 1 0.5
+7 1 0.5
+8 1 0.5
+9 1 0.5
+10 1 0.5
+11 1 0.5
+12 1 0.5
+13 1 0.5
+14 1 0.5
+15 1 0.5
+16 1 0.5
+17 1 0.5
+18 1 0.5
+19 1 0.5
+20 1 0.5
+21 1 0.5
+22 1 0.5
+23 1 0.5
+24 1 0.5
+25 1 0.5
+0 2 0.5
+1 2 0.5
+24 2 0.5
+25 2 0.5
+0 3 0.5
+1 3 0.5
+24 3 0.5
+25 3 0.5
+0 4 0.5
+1 4 0.5
+24 4 0.5
+25 4 0.5
+0 5 0.5
+1 5 0.5
+24 5 0.5
+25 5 0.5
+0 6 0.5
+1 6 0.5
+7.9765625 6.8515625 0.5
+24 6 0.5
+25 6 0.5
+0 7 0.5
+1 7 0.5
+24 7 0.5
+25 7 0.5
+0 8 0.5
+1 8 0.5
+24 8 0.5
+25 8 0.5
+0 9 0.5
+1 9 0.5
+24 9 0.5
+25 9 0.5
+0 10 0.5
+1 10 0.5
+16.6171875 7.7265625 0.5
+24 10 0.5
+25 10 0.5
+0 11 0.5
+1 11 0.5
+2 11 0.5
+3 11 0.5
+4 11 0.5
+5 11 0.5
+6 11 0.5
+7 11 0.5
+8 11 0.5
+9 11 0.5
+10 11 0.5
+11 11 0.5
+12 11 0.5
+13 11 0.5
+14 11 0.5
+15 11 0.5
+16 11 0.5
+17 11 0.5
+18 11 0.5
+19 11 0.5
+20 11 0.5
+21 11 0.5
+22 11 0.5
+23 11 0.5
+24 11 0.5
+25 11 0.5
+0 12 0.5
+1 12 0.5
+2 12 0.5
+3 12 0.5
+4 12 0.5
+5 12 0.5
+6 12 0.5
+7 12 0.5
+8 12 0.5
+9 12 0.5
+10 12 0.5
+11 12 0.5
+12 12 0.5
+13 12 0.5
+14 12 0.5
+15 12 0.5
+16 12 0.5
+17 12 0.5
+18 12 0.5
+19 12 0.5
+20 12 0.5
+21 12 0.5
+22 12 0.5
+23 12 0.5
+24 12 0.5
+25 12 0.5
+0 13 0.5
+1 13 0.5
+2 13 0.5
+3 13 0.5
+4 13 0.5
+5 13 0.5
+6 13 0.5
+7 13 0.5
+8 13 0.5
+9 13 0.5
+10 13 0.5
+11 13 0.5
+12 13 0.5
+13 13 0.5
+14 13 0.5
+15 13 0.5
+16 13 0.5
+17 13 0.5
+18 13 0.5
+19 13 0.5
+20 13 0.5
+21 13 0.5
+22 13 0.5
+23 13 0.5
+24 13 0.5
+25 13 0.5
+0 14 0.5
+1 14 0.5
+2 14 0.5
+3 14 0.5
+4 14 0.5
+5 14 0.5
+6 14 0.5
+7 14 0.5
+8 14 0.5
+9 14 0.5
+10 14 0.5
+11 14 0.5
+12 14 0.5
+13 14 0.5
+14 14 0.5
+15 14 0.5
+16 14 0.5
+17 14 0.5
+18 14 0.5
+19 14 0.5
+20 14 0.5
+21 14 0.5
+22 14 0.5
+23 14 0.5
+24 14 0.5
+25 14 0.5
+0 15 0.5
+1 15 0.5
+24 15 0.5
+25 15 0.5
+0 16 0.5
+1 16 0.5
+24 16 0.5
+25 16 0.5
+0 17 0.5
+1 17 0.5
+24 17 0.5
+25 17 0.5
+0 18 0.5
+1 18 0.5
+24 18 0.5
+25 18 0.5
+0 19 0.5
+1 19 0.5
+7.9765625 19.8515625 0.5
+24 19 0.5
+25 19 0.5
+0 20 0.5
+1 20 0.5
+24 20 0.5
+25 20 0.5
+0 21 0.5
+1 21 0.5
+24 21 0.5
+25 21 0.5
+0 22 0.5
+1 22 0.5
+24 22 0.5
+25 22 0.5
+0 23 0.5
+1 23 0.5
+16.6171875 20.7265625 0.5
+24 23 0.5
+25 23 0.5
+0 24 0.5
+1 24 0.5
+2 24 0.5
+3 24 0.5
+4 24 0.5
+5 24 0.5
+6 24 0.5
+7 24 0.5
+8 24 0.5
+9 24 0.5
+10 24 0.5
+11 24 0.5
+12 24 0.5
+13 24 0.5
+14 24 0.5
+15 24 0.5
+16 24 0.5
+17 24 0.5
+18 24 0.5
+19 24 0.5
+20 24 0.5
+21 24 0.5
+22 24 0.5
+23 24 0.5
+24 24 0.5
+25 24 0.5
+0 25 0.5
+1 25 0.5
+2 25 0.5
+3 25 0.5
+4 25 0.5
+5 25 0.5
+6 25 0.5
+7 25 0.5
+8 25 0.5
+9 25 0.5
+10 25 0.5
+11 25 0.5
+12 25 0.5
+13 25 0.5
+14 25 0.5
+15 25 0.5
+16 25 0.5
+17 25 0.5
+18 25 0.5
+19 25 0.5
+20 25 0.5
+21 25 0.5
+22 25 0.5
+23 25 0.5
+24 25 0.5
+25 25 0.5
+POLYGONS 962 3848
+3 199 200 226
+3 199 226 225
+3 0 27 1
+3 0 26 27
+3 0 199 26
+3 199 225 26
+3 0 1 200
+3 0 200 199
+3 200 201 227
+3 200 227 226
+3 1 28 2
+3 1 27 28
+3 1 2 201
+3 1 201 200
+3 201 202 228
+3 201 228 227
+3 2 29 3
+3 2 28 29
+3 2 3 202
+3 2 202 201
+3 202 203 229
+3 202 229 228
+3 3 30 4
+3 3 29 30
+3 3 4 203
+3 3 203 202
+3 203 204 230
+3 203 230 229
+3 4 31 5
+3 4 30 31
+3 4 5 204
+3 4 204 203
+3 204 205 231
+3 204 231 230
+3 5 32 6
+3 5 31 32
+3 5 6 205
+3 5 205 204
+3 205 206 232
+3 205 232 231
+3 6 33 7
+3 6 32 33
+3 6 7 206
+3 6 206 205
+3 206 207 233
+3 206 233 232
+3 7 34 8
+3 7 33 34
+3 7 8 207
+3 7 207 206
+3 207 208 234
+3 207 234 233
+3 8 35 9
+3 8 34 35
+3 8 9 208
+3 8 208 207
+3 208 209 235
+3 208 235 234
+3 9 36 10
+3 9 35 36
+3 9 10 209
+3 9 209 208
+3 209 210 236
+3 209 236 235
+3 10 37 11
+3 10 36 37
+3 10 11 210
+3 10 210 209
+3 210 211 237
+3 210 237 236
+3 11 38 12
+3 11 37 38
+3 11 12 211
+3 11 211 210
+3 211 212 238
+3 211 238 237
+3 12 39 13
+3 12 38 39
+3 12 13 212
+3 12 212 211
+3 212 213 239
+3 212 239 238
+3 13 40 14
+3 13 39 40
+3 13 14 213
+3 13 213 212
+3 213 214 240
+3 213 240 239
+3 14 41 15
+3 14 40 41
+3 14 15 214
+3 14 214 213
+3 214 215 241
+3 214 241 240
+3 15 42 16
+3 15 41 42
+3 15 16 215
+3 15 215 214
+3 215 216 242
+3 215 242 241
+3 16 43 17
+3 16 42 43
+3 16 17 216
+3 16 216 215
+3 216 217 243
+3 216 243 242
+3 17 44 18
+3 17 43 44
+3 17 18 217
+3 17 217 216
+3 217 218 244
+3 217 244 243
+3 18 45 19
+3 18 44 45
+3 18 19 218
+3 18 218 217
+3 218 219 245
+3 218 245 244
+3 19 46 20
+3 19 45 46
+3 19 20 219
+3 19 219 218
+3 219 220 246
+3 219 246 245
+3 20 47 21
+3 20 46 47
+3 20 21 220
+3 20 220 219
+3 220 221 247
+3 220 247 246
+3 21 48 22
+3 21 47 48
+3 21 22 221
+3 21 221 220
+3 221 222 248
+3 221 248 247
+3 22 49 23
+3 22 48 49
+3 22 23 222
+3 22 222 221
+3 222 223 249
+3 222 249 248
+3 23 50 24
+3 23 49 50
+3 23 24 223
+3 23 223 222
+3 223 224 250
+3 223 250 249
+3 24 51 25
+3 24 50 51
+3 51 224 25
+3 224 51 250
+3 24 25 224
+3 24 224 223
+3 225 226 252
+3 225 252 251
+3 26 53 27
+3 26 52 53
+3 26 225 52
+3 225 251 52
+3 226 227 269
+3 226 269 252
+3 27 80 28
+3 27 53 80
+3 227 228 269
+3 28 80 29
+3 228 229 269
+3 29 80 30
+3 229 230 269
+3 30 80 31
+3 230 231 269
+3 31 80 32
+3 231 232 269
+3 32 80 33
+3 232 233 269
+3 33 80 34
+3 233 234 269
+3 34 80 35
+3 234 235 286
+3 234 286 269
+3 35 58 36
+3 35 80 58
+3 235 236 286
+3 36 58 37
+3 236 237 286
+3 37 58 38
+3 237 238 286
+3 38 58 39
+3 238 239 286
+3 39 58 40
+3 239 240 286
+3 40 58 41
+3 240 241 286
+3 41 58 42
+3 241 242 286
+3 42 58 43
+3 242 243 286
+3 43 67 44
+3 43 58 67
+3 243 244 286
+3 44 67 45
+3 244 245 286
+3 45 67 46
+3 245 246 286
+3 46 67 47
+3 246 247 286
+3 47 67 48
+3 247 248 286
+3 48 67 49
+3 248 249 253
+3 248 253 286
+3 49 54 50
+3 49 67 54
+3 249 250 254
+3 249 254 253
+3 50 55 51
+3 50 54 55
+3 55 250 51
+3 250 55 254
+3 251 252 256
+3 251 256 255
+3 52 57 53
+3 52 56 57
+3 52 251 56
+3 251 255 56
+3 252 269 256
+3 53 57 80
+3 80 93 58
+3 286 253 257
+3 67 59 54
+3 253 254 258
+3 253 258 257
+3 54 60 55
+3 54 59 60
+3 60 254 55
+3 254 60 258
+3 255 256 260
+3 255 260 259
+3 56 62 57
+3 56 61 62
+3 56 255 61
+3 255 259 61
+3 256 269 260
+3 57 62 80
+3 286 257 261
+3 67 63 59
+3 257 258 262
+3 257 262 261
+3 59 64 60
+3 59 63 64
+3 64 258 60
+3 258 64 262
+3 259 260 264
+3 259 264 263
+3 61 66 62
+3 61 65 66
+3 61 259 65
+3 259 263 65
+3 260 269 264
+3 62 66 80
+3 286 261 265
+3 67 68 63
+3 261 262 266
+3 261 266 265
+3 63 69 64
+3 63 68 69
+3 69 262 64
+3 262 69 266
+3 263 264 268
+3 263 268 267
+3 65 71 66
+3 65 70 71
+3 65 263 70
+3 263 267 70
+3 264 269 268
+3 66 71 80
+3 286 265 270
+3 67 72 68
+3 265 266 271
+3 265 271 270
+3 68 73 69
+3 68 72 73
+3 73 266 69
+3 266 73 271
+3 267 268 273
+3 267 273 272
+3 70 75 71
+3 70 74 75
+3 70 267 74
+3 267 272 74
+3 268 269 273
+3 71 75 80
+3 286 270 274
+3 67 76 72
+3 67 58 76
+3 270 271 275
+3 270 275 274
+3 72 77 73
+3 72 76 77
+3 77 271 73
+3 271 77 275
+3 272 273 277
+3 272 277 276
+3 74 79 75
+3 74 78 79
+3 74 272 78
+3 272 276 78
+3 273 269 277
+3 75 79 80
+3 286 274 278
+3 58 81 76
+3 274 275 279
+3 274 279 278
+3 76 82 77
+3 76 81 82
+3 82 275 77
+3 275 82 279
+3 276 277 281
+3 276 281 280
+3 78 84 79
+3 78 83 84
+3 78 276 83
+3 276 280 83
+3 277 269 281
+3 79 130 80
+3 79 84 130
+3 286 278 282
+3 58 85 81
+3 278 279 283
+3 278 283 282
+3 81 86 82
+3 81 85 86
+3 86 279 82
+3 279 86 283
+3 280 281 285
+3 280 285 284
+3 83 88 84
+3 83 87 88
+3 83 280 87
+3 280 284 87
+3 281 269 285
+3 84 88 130
+3 286 282 287
+3 58 89 85
+3 282 283 288
+3 282 288 287
+3 85 90 86
+3 85 89 90
+3 90 283 86
+3 283 90 288
+3 284 285 290
+3 284 290 289
+3 87 92 88
+3 87 91 92
+3 87 284 91
+3 284 289 91
+3 285 269 291
+3 285 291 290
+3 88 92 130
+3 269 292 291
+3 269 293 292
+3 269 294 293
+3 269 295 294
+3 269 296 295
+3 269 297 296
+3 269 298 297
+3 269 299 298
+3 269 300 299
+3 269 301 300
+3 269 302 301
+3 269 303 302
+3 269 286 304
+3 269 304 303
+3 286 305 304
+3 286 306 305
+3 286 307 306
+3 286 308 307
+3 286 309 308
+3 286 310 309
+3 286 311 310
+3 286 312 311
+3 286 287 313
+3 286 313 312
+3 58 94 89
+3 58 93 94
+3 287 288 314
+3 287 314 313
+3 89 95 90
+3 89 94 95
+3 95 288 90
+3 288 95 314
+3 289 290 316
+3 289 316 315
+3 91 97 92
+3 91 96 97
+3 91 289 96
+3 289 315 96
+3 290 291 317
+3 290 317 316
+3 92 97 130
+3 291 292 318
+3 291 318 317
+3 292 293 319
+3 292 319 318
+3 293 294 320
+3 293 320 319
+3 294 295 321
+3 294 321 320
+3 295 296 322
+3 295 322 321
+3 296 297 323
+3 296 323 322
+3 297 298 324
+3 297 324 323
+3 298 299 325
+3 298 325 324
+3 299 300 326
+3 299 326 325
+3 300 301 327
+3 300 327 326
+3 301 302 328
+3 301 328 327
+3 302 303 329
+3 302 329 328
+3 303 304 330
+3 303 330 329
+3 304 305 331
+3 304 331 330
+3 305 306 332
+3 305 332 331
+3 306 307 333
+3 306 333 332
+3 307 308 334
+3 307 334 333
+3 308 309 335
+3 308 335 334
+3 309 310 336
+3 309 336 335
+3 310 311 337
+3 310 337 336
+3 311 312 338
+3 311 338 337
+3 312 313 339
+3 312 339 338
+3 93 98 94
+3 313 314 340
+3 313 340 339
+3 94 99 95
+3 94 98 99
+3 99 314 95
+3 314 99 340
+3 315 316 342
+3 315 342 341
+3 96 101 97
+3 96 100 101
+3 96 315 100
+3 315 341 100
+3 316 317 343
+3 316 343 342
+3 97 101 130
+3 317 318 344
+3 317 344 343
+3 318 319 345
+3 318 345 344
+3 319 320 346
+3 319 346 345
+3 320 321 347
+3 320 347 346
+3 321 322 348
+3 321 348 347
+3 322 323 349
+3 322 349 348
+3 323 324 350
+3 323 350 349
+3 324 325 351
+3 324 351 350
+3 325 326 352
+3 325 352 351
+3 326 327 353
+3 326 353 352
+3 327 328 354
+3 327 354 353
+3 328 329 355
+3 328 355 354
+3 329 330 356
+3 329 356 355
+3 330 331 357
+3 330 357 356
+3 331 332 358
+3 331 358 357
+3 332 333 359
+3 332 359 358
+3 333 334 360
+3 333 360 359
+3 334 335 361
+3 334 361 360
+3 335 336 362
+3 335 362 361
+3 336 337 363
+3 336 363 362
+3 337 338 364
+3 337 364 363
+3 338 339 365
+3 338 365 364
+3 93 102 98
+3 339 340 366
+3 339 366 365
+3 98 103 99
+3 98 102 103
+3 103 340 99
+3 340 103 366
+3 341 342 368
+3 341 368 367
+3 100 105 101
+3 100 104 105
+3 100 341 104
+3 341 367 104
+3 342 343 369
+3 342 369 368
+3 101 105 130
+3 343 344 370
+3 343 370 369
+3 344 345 371
+3 344 371 370
+3 345 346 372
+3 345 372 371
+3 346 347 373
+3 346 373 372
+3 347 348 374
+3 347 374 373
+3 348 349 375
+3 348 375 374
+3 349 350 376
+3 349 376 375
+3 350 351 377
+3 350 377 376
+3 351 352 378
+3 351 378 377
+3 352 353 379
+3 352 379 378
+3 353 354 380
+3 353 380 379
+3 354 355 381
+3 354 381 380
+3 355 356 382
+3 355 382 381
+3 356 357 383
+3 356 383 382
+3 357 358 384
+3 357 384 383
+3 358 359 385
+3 358 385 384
+3 359 360 386
+3 359 386 385
+3 360 361 387
+3 360 387 386
+3 361 362 388
+3 361 388 387
+3 362 363 389
+3 362 389 388
+3 363 364 390
+3 363 390 389
+3 364 365 391
+3 364 391 390
+3 93 106 102
+3 365 366 392
+3 365 392 391
+3 102 107 103
+3 102 106 107
+3 107 366 103
+3 366 107 392
+3 367 368 394
+3 367 394 393
+3 104 109 105
+3 104 108 109
+3 104 367 108
+3 367 393 108
+3 368 369 411
+3 368 411 394
+3 105 109 130
+3 369 370 411
+3 370 371 411
+3 371 372 411
+3 372 373 411
+3 373 374 411
+3 374 375 411
+3 375 376 411
+3 376 377 428
+3 376 428 411
+3 377 378 428
+3 378 379 428
+3 379 380 428
+3 380 381 428
+3 381 382 428
+3 382 383 428
+3 383 384 428
+3 384 385 428
+3 385 386 428
+3 386 387 428
+3 387 388 428
+3 388 389 428
+3 389 390 428
+3 390 391 395
+3 390 395 428
+3 93 110 106
+3 391 392 396
+3 391 396 395
+3 106 111 107
+3 106 110 111
+3 111 392 107
+3 392 111 396
+3 393 394 398
+3 393 398 397
+3 108 113 109
+3 108 112 113
+3 108 393 112
+3 393 397 112
+3 394 411 398
+3 109 113 130
+3 428 395 399
+3 93 114 110
+3 395 396 400
+3 395 400 399
+3 110 115 111
+3 110 114 115
+3 115 396 111
+3 396 115 400
+3 397 398 402
+3 397 402 401
+3 112 117 113
+3 112 116 117
+3 112 397 116
+3 397 401 116
+3 398 411 402
+3 113 117 130
+3 80 130 93
+3 428 399 403
+3 93 118 114
+3 399 400 404
+3 399 404 403
+3 114 119 115
+3 114 118 119
+3 119 400 115
+3 400 119 404
+3 401 402 406
+3 401 406 405
+3 116 121 117
+3 116 120 121
+3 116 401 120
+3 401 405 120
+3 402 411 406
+3 117 121 130
+3 428 403 407
+3 93 122 118
+3 403 404 408
+3 403 408 407
+3 118 123 119
+3 118 122 123
+3 123 404 119
+3 404 123 408
+3 405 406 410
+3 405 410 409
+3 120 125 121
+3 120 124 125
+3 120 405 124
+3 405 409 124
+3 406 411 410
+3 121 125 130
+3 428 407 412
+3 93 126 122
+3 407 408 413
+3 407 413 412
+3 122 127 123
+3 122 126 127
+3 127 408 123
+3 408 127 413
+3 409 410 415
+3 409 415 414
+3 124 129 125
+3 124 128 129
+3 124 409 128
+3 409 414 128
+3 410 411 415
+3 125 129 130
+3 428 412 416
+3 93 131 126
+3 412 413 417
+3 412 417 416
+3 126 132 127
+3 126 131 132
+3 132 413 127
+3 413 132 417
+3 414 415 419
+3 414 419 418
+3 128 134 129
+3 128 133 134
+3 128 414 133
+3 414 418 133
+3 415 411 419
+3 129 135 130
+3 129 134 135
+3 428 416 420
+3 93 136 131
+3 416 417 421
+3 416 421 420
+3 131 137 132
+3 131 136 137
+3 137 417 132
+3 417 137 421
+3 418 419 423
+3 418 423 422
+3 133 139 134
+3 133 138 139
+3 133 418 138
+3 418 422 138
+3 419 411 423
+3 134 139 135
+3 135 140 130
+3 130 140 93
+3 428 420 424
+3 93 141 136
+3 420 421 425
+3 420 425 424
+3 136 142 137
+3 136 141 142
+3 142 421 137
+3 421 142 425
+3 422 423 427
+3 422 427 426
+3 138 144 139
+3 138 143 144
+3 138 422 143
+3 422 426 143
+3 423 411 427
+3 139 144 135
+3 428 424 429
+3 93 145 141
+3 424 425 430
+3 424 430 429
+3 141 146 142
+3 141 145 146
+3 146 425 142
+3 425 146 430
+3 426 427 432
+3 426 432 431
+3 143 148 144
+3 143 147 148
+3 143 426 147
+3 426 431 147
+3 427 411 433
+3 427 433 432
+3 144 149 135
+3 144 148 149
+3 411 434 433
+3 135 149 150
+3 411 435 434
+3 135 150 151
+3 411 436 435
+3 135 151 152
+3 411 437 436
+3 135 152 153
+3 411 438 437
+3 135 154 140
+3 135 153 154
+3 411 439 438
+3 140 154 155
+3 411 440 439
+3 140 155 156
+3 411 441 440
+3 140 156 157
+3 411 442 441
+3 140 157 158
+3 411 443 442
+3 140 158 159
+3 411 444 443
+3 140 160 93
+3 140 159 160
+3 411 445 444
+3 93 160 161
+3 411 428 446
+3 411 446 445
+3 93 161 162
+3 428 447 446
+3 93 162 163
+3 428 448 447
+3 93 163 164
+3 428 449 448
+3 93 164 165
+3 428 450 449
+3 93 165 166
+3 428 451 450
+3 93 166 167
+3 428 452 451
+3 93 167 168
+3 428 453 452
+3 93 168 169
+3 428 454 453
+3 93 169 170
+3 428 429 455
+3 428 455 454
+3 93 171 145
+3 93 170 171
+3 429 430 456
+3 429 456 455
+3 145 172 146
+3 145 171 172
+3 172 430 146
+3 430 172 456
+3 431 432 458
+3 431 458 457
+3 147 174 148
+3 147 173 174
+3 147 431 173
+3 431 457 173
+3 458 174 173
+3 457 458 173
+3 432 433 459
+3 432 459 458
+3 148 175 149
+3 148 174 175
+3 459 175 174
+3 458 459 174
+3 433 434 460
+3 433 460 459
+3 149 176 150
+3 149 175 176
+3 460 176 175
+3 459 460 175
+3 434 435 461
+3 434 461 460
+3 150 177 151
+3 150 176 177
+3 461 177 176
+3 460 461 176
+3 435 436 462
+3 435 462 461
+3 151 178 152
+3 151 177 178
+3 462 178 177
+3 461 462 177
+3 436 437 463
+3 436 463 462
+3 152 179 153
+3 152 178 179
+3 463 179 178
+3 462 463 178
+3 437 438 464
+3 437 464 463
+3 153 180 154
+3 153 179 180
+3 464 180 179
+3 463 464 179
+3 438 439 465
+3 438 465 464
+3 154 181 155
+3 154 180 181
+3 465 181 180
+3 464 465 180
+3 439 440 466
+3 439 466 465
+3 155 182 156
+3 155 181 182
+3 466 182 181
+3 465 466 181
+3 440 441 467
+3 440 467 466
+3 156 183 157
+3 156 182 183
+3 467 183 182
+3 466 467 182
+3 441 442 468
+3 441 468 467
+3 157 184 158
+3 157 183 184
+3 468 184 183
+3 467 468 183
+3 442 443 469
+3 442 469 468
+3 158 185 159
+3 158 184 185
+3 469 185 184
+3 468 469 184
+3 443 444 470
+3 443 470 469
+3 159 186 160
+3 159 185 186
+3 470 186 185
+3 469 470 185
+3 444 445 471
+3 444 471 470
+3 160 187 161
+3 160 186 187
+3 471 187 186
+3 470 471 186
+3 445 446 472
+3 445 472 471
+3 161 188 162
+3 161 187 188
+3 472 188 187
+3 471 472 187
+3 446 447 473
+3 446 473 472
+3 162 189 163
+3 162 188 189
+3 473 189 188
+3 472 473 188
+3 447 448 474
+3 447 474 473
+3 163 190 164
+3 163 189 190
+3 474 190 189
+3 473 474 189
+3 448 449 475
+3 448 475 474
+3 164 191 165
+3 164 190 191
+3 475 191 190
+3 474 475 190
+3 449 450 476
+3 449 476 475
+3 165 192 166
+3 165 191 192
+3 476 192 191
+3 475 476 191
+3 450 451 477
+3 450 477 476
+3 166 193 167
+3 166 192 193
+3 477 193 192
+3 476 477 192
+3 451 452 478
+3 451 478 477
+3 167 194 168
+3 167 193 194
+3 478 194 193
+3 477 478 193
+3 452 453 479
+3 452 479 478
+3 168 195 169
+3 168 194 195
+3 479 195 194
+3 478 479 194
+3 453 454 480
+3 453 480 479
+3 169 196 170
+3 169 195 196
+3 480 196 195
+3 479 480 195
+3 454 455 481
+3 454 481 480
+3 170 197 171
+3 170 196 197
+3 481 197 196
+3 480 481 196
+3 455 456 482
+3 455 482 481
+3 171 198 172
+3 171 197 198
+3 198 456 172
+3 456 198 482
+3 482 198 197
+3 481 482 197
+CELL_DATA 962
+SCALARS patch_index int 1
+LOOKUP_TABLE default
+5
+5
+1
+1
+2
+2
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+5
+1
+5
+1
+5
+1
+5
+1
+5
+1
+5
+1
+5
+1
+5
+5
+1
+1
+5
+1
+5
+1
+5
+1
+5
+1
+5
+1
+5
+1
+5
+1
+5
+1
+1
+5
+1
+5
+1
+5
+1
+5
+1
+5
+1
+5
+5
+1
+1
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+1
+1
+5
+1
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+1
+5
+1
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+1
+5
+1
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+1
+5
+1
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+1
+5
+1
+1
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+1
+5
+1
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+1
+1
+5
+1
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+1
+5
+1
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+1
+1
+5
+5
+1
+1
+2
+2
+5
+5
+1
+1
+2
+2
+5
+5
+1
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+5
+1
+5
+5
+1
+1
+2
+2
+4
+4
+1
+1
+2
+2
+4
+4
+1
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+4
+1
+4
+4
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+1
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+3
+1
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+1
+3
+1
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+1
+1
+3
+1
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+1
+3
+1
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+1
+3
+1
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+1
+3
+1
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+1
+1
+3
+1
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+1
+1
+1
+3
+1
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+1
+3
+1
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+3
+1
+3
+1
+3
+1
+3
+1
+3
+1
+1
+3
+1
+3
+1
+3
+1
+3
+1
+3
+1
+3
+1
+1
+3
+1
+3
+3
+1
+3
+1
+3
+1
+3
+1
+3
+1
+3
+1
+3
+1
+3
+1
+3
+1
+3
+3
+1
+1
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+3
+3
+1
+1
+2
+2
+2
+2
diff --git a/test/input/tilted_v_with_reservoir_initial_pressure.pfb b/test/input/tilted_v_with_reservoir_initial_pressure.pfb
new file mode 100644
index 000000000..4d13e788e
Binary files /dev/null and b/test/input/tilted_v_with_reservoir_initial_pressure.pfb differ
diff --git a/test/input/tilted_v_with_reservoir_initial_pressure.pfb.dist b/test/input/tilted_v_with_reservoir_initial_pressure.pfb.dist
new file mode 100644
index 000000000..573541ac9
--- /dev/null
+++ b/test/input/tilted_v_with_reservoir_initial_pressure.pfb.dist
@@ -0,0 +1 @@
+0
diff --git a/test/python/CMakeLists.txt b/test/python/CMakeLists.txt
index a546a44e3..18afee14b 100644
--- a/test/python/CMakeLists.txt
+++ b/test/python/CMakeLists.txt
@@ -2,6 +2,7 @@ include(ParflowTest)
 
 set(TESTS
   default_single
+  default_single_read_porosity
   default_richards_wells
   forsyth2
   harvey_flow
@@ -11,11 +12,19 @@ set(TESTS
   crater2D_vangtable_linear
   small_domain
   richards_hydrostatic_equilibrium
+  indicator_field
+  LW_surface_press
+  channelwidth
+  overland_tiltedV_MGSemiTest
 )
 
 if(${PARFLOW_HAVE_HYPRE})
   list(APPEND TESTS
+    bc_pressure_file
+    bc_flux_file
+    Dirichlet    
     default_richards
+    default_richards_restart
     default_richards_nocluster
     default_overland
     octree-simple
@@ -24,6 +33,8 @@ if(${PARFLOW_HAVE_HYPRE})
     LW_var_dz_spinup
     terrain_following_grid_overland
     var_dz_1D
+    tilted_v_with_reservoir_overland_flow
+    tilted_v_with_reservoir_overland_kinematic
     pfmg
     pfmg_galerkin
     smg
@@ -37,7 +48,11 @@ if(${PARFLOW_HAVE_HYPRE})
     richards_FBx
     richards_FBy
     richards_box_proctest
-    richards_box_proctest_vardz)
+    richards_box_proctest_vardz
+    water_balance_y
+    water_balance_x
+    water_balance_x.hardflow.nojac
+    water_balance_x.hardflow.jac)
 endif()
 
 if(${PARFLOW_HAVE_SILO})
@@ -47,11 +62,8 @@ if(${PARFLOW_HAVE_SILO})
 
   if(${PARFLOW_HAVE_HYPRE})
     list(APPEND TESTS
-      water_balance_y
-      water_balance_x
-      water_balance_x.hardflow.nojac
-      water_balance_x.hardflow.jac
-      default_richards_with_silo)
+      # default_richards_with_silo
+      )
   endif()
 endif()
 
@@ -59,7 +71,8 @@ if(${PARFLOW_HAVE_NETCDF})
   if(${PARFLOW_HAVE_HYPRE})
     #This test is failing on several platforms
     list(APPEND TESTS
-#    	default_richards_with_netcdf
+      #    	default_richards_with_netcdf
+      default_richards_restart_netcdf
         )
   endif()
 endif()
@@ -80,11 +93,19 @@ list(APPEND PARALLEL_2DTOPO_TESTS "")
 
 if ( ${PARFLOW_AMPS_LAYER} IN_LIST PARFLOW_AMPS_LAYER_REQUIRE_MPI )
   list(APPEND PARALLEL_3DTOPO_TESTS
-    default_single)
+    default_single
+    default_single_read_porosity
+    )
 
   if(${PARFLOW_HAVE_HYPRE})
     list(APPEND PARALLEL_3DTOPO_TESTS
-      default_richards)
+      default_richards
+      default_richards_restart)
+
+    if(${PARFLOW_HAVE_NETCDF})
+      list(APPEND PARALLEL_3DTOPO_TESTS
+	default_richards_restart_netcdf)
+    endif()
 
     list(APPEND PARALLEL_2DTOPO_TESTS
       default_overland
@@ -93,7 +114,9 @@ if ( ${PARFLOW_AMPS_LAYER} IN_LIST PARFLOW_AMPS_LAYER_REQUIRE_MPI )
       LW_var_dz
       LW_var_dz_spinup
       overland_slopingslab_KWE
-      richards_box_proctest_vardz)
+      richards_box_proctest_vardz
+            reservoir_mpi_test
+    )
 
     if(${PARFLOW_HAVE_SILO})
       list(APPEND PARALLEL_2DTOPO_TESTS
@@ -129,7 +152,9 @@ foreach(inputfile ${PARALLEL_3DTOPO_TESTS})
   pf_add_py_parallel_test(${inputfile} 1 4 1)
   pf_add_py_parallel_test(${inputfile} 4 1 1)
   pf_add_py_parallel_test(${inputfile} 2 2 2)
-  pf_add_py_parallel_test(${inputfile} 3 3 3)
+  if(NOT( ((${PARFLOW_HAVE_CUDA}) OR (${PARFLOW_HAVE_KOKKOS}) OR (${PARFLOW_HAVE_OMP})) ))
+    pf_add_py_parallel_test(${inputfile} 3 3 3)
+  endif()
 
 endforeach()
 
@@ -141,5 +166,5 @@ add_subdirectory (key_validation)
 
 if (${PARFLOW_HAVE_CLM})
   add_subdirectory (clm)
-#  add_subdirectory (washita)
+  add_subdirectory (washita)
 endif(${PARFLOW_HAVE_CLM})
diff --git a/test/python/Dirichlet.py b/test/python/Dirichlet.py
new file mode 100644
index 000000000..a8b1e39de
--- /dev/null
+++ b/test/python/Dirichlet.py
@@ -0,0 +1,362 @@
+import sys
+from parflow import Run
+from parflow.tools.fs import cp, mkdir, chdir, get_absolute_path, rm
+from parflow.tools.compare import pf_test_file, pf_test_file_with_abs
+
+run_name = "Dirichlet"
+dirichlet = Run(run_name, __file__)
+
+# -----------------------------------------------------------------------------
+# File input version number
+# -----------------------------------------------------------------------------
+dirichlet.FileVersion = 4
+
+# -----------------------------------------------------------------------------
+# Process Topology
+# -----------------------------------------------------------------------------
+
+dirichlet.Process.Topology.P = 1
+dirichlet.Process.Topology.Q = 1
+dirichlet.Process.Topology.R = 1
+
+# -----------------------------------------------------------------------------
+# Computational Grid
+# -----------------------------------------------------------------------------
+dirichlet.ComputationalGrid.Lower.X = 0.0
+dirichlet.ComputationalGrid.Lower.Y = 0.0
+dirichlet.ComputationalGrid.Lower.Z = 0.0
+
+dirichlet.ComputationalGrid.DX = 1.0
+dirichlet.ComputationalGrid.DY = 1.0
+dirichlet.ComputationalGrid.DZ = 0.1
+
+dirichlet.ComputationalGrid.NX = 1
+dirichlet.ComputationalGrid.NY = 1
+dirichlet.ComputationalGrid.NZ = 10
+
+# -----------------------------------------------------------------------------
+# The Names of the GeomInputs
+# -----------------------------------------------------------------------------
+dirichlet.GeomInput.Names = "domain_input"
+
+
+# -----------------------------------------------------------------------------
+# Domain Geometry Input
+# -----------------------------------------------------------------------------
+dirichlet.GeomInput.domain_input.InputType = "Box"
+dirichlet.GeomInput.domain_input.GeomName = "domain"
+
+# -----------------------------------------------------------------------------
+# Domain Geometry
+# -----------------------------------------------------------------------------
+dirichlet.Geom.domain.Lower.X = 0.0
+dirichlet.Geom.domain.Lower.Y = 0.0
+dirichlet.Geom.domain.Lower.Z = 0.0
+
+dirichlet.Geom.domain.Upper.X = 1.0
+dirichlet.Geom.domain.Upper.Y = 1.0
+dirichlet.Geom.domain.Upper.Z = 1.0
+
+dirichlet.Geom.domain.Patches = "left right front back bottom top"
+
+# -----------------------------------------------------------------------------
+# Perm
+# -----------------------------------------------------------------------------
+dirichlet.Geom.Perm.Names = "domain"
+
+dirichlet.Geom.domain.Perm.Type = "Constant"
+dirichlet.Geom.domain.Perm.Value = 4.0
+
+dirichlet.Perm.TensorType = "TensorByGeom"
+
+dirichlet.Geom.Perm.TensorByGeom.Names = "domain"
+
+dirichlet.Geom.domain.Perm.TensorValX = 1.0
+dirichlet.Geom.domain.Perm.TensorValY = 1.0
+dirichlet.Geom.domain.Perm.TensorValZ = 1.0
+
+# -----------------------------------------------------------------------------
+# Specific Storage
+# -----------------------------------------------------------------------------
+# specific storage does not figure into the impes (fully sat) case but we still
+# need a key for it
+
+dirichlet.SpecificStorage.Type = "Constant"
+dirichlet.SpecificStorage.GeomNames = ""
+dirichlet.Geom.domain.SpecificStorage.Value = 1.0e-4
+
+# -----------------------------------------------------------------------------
+# Phases
+# -----------------------------------------------------------------------------
+
+dirichlet.Phase.Names = "water"
+
+dirichlet.Phase.water.Density.Type = "Constant"
+dirichlet.Phase.water.Density.Value = 1.0
+
+dirichlet.Phase.water.Viscosity.Type = "Constant"
+dirichlet.Phase.water.Viscosity.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Contaminants
+# -----------------------------------------------------------------------------
+dirichlet.Contaminants.Names = ""
+
+# -----------------------------------------------------------------------------
+# Retardation
+# -----------------------------------------------------------------------------
+dirichlet.Geom.Retardation.GeomNames = ""
+
+# -----------------------------------------------------------------------------
+# Gravity
+# -----------------------------------------------------------------------------
+
+dirichlet.Gravity = 1.0
+
+# -----------------------------------------------------------------------------
+# Setup timing info
+# -----------------------------------------------------------------------------
+
+dirichlet.TimingInfo.BaseUnit = 1.0
+dirichlet.TimingInfo.StartCount = 0
+dirichlet.TimingInfo.StartTime = 0.0
+dirichlet.TimingInfo.StopTime = 1.0
+dirichlet.TimingInfo.DumpInterval = -1
+dirichlet.TimeStep.Type = "Constant"
+dirichlet.TimeStep.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Porosity
+# -----------------------------------------------------------------------------
+
+dirichlet.Geom.Porosity.GeomNames = "domain"
+
+dirichlet.Geom.domain.Porosity.Type = "Constant"
+dirichlet.Geom.domain.Porosity.Value = 0.5
+
+# -----------------------------------------------------------------------------
+# Domain
+# -----------------------------------------------------------------------------
+dirichlet.Domain.GeomName = "domain"
+
+# -----------------------------------------------------------------------------
+# Relative Permeability
+# -----------------------------------------------------------------------------
+
+dirichlet.Phase.RelPerm.Type = "VanGenuchten"
+dirichlet.Phase.RelPerm.GeomNames = "domain"
+dirichlet.Geom.domain.RelPerm.Alpha = 1.0
+dirichlet.Geom.domain.RelPerm.N = 2.0
+
+# ---------------------------------------------------------
+# Saturation
+# ---------------------------------------------------------
+
+dirichlet.Phase.Saturation.Type = "VanGenuchten"
+dirichlet.Phase.Saturation.GeomNames = "domain"
+dirichlet.Geom.domain.Saturation.Alpha = 1.0
+dirichlet.Geom.domain.Saturation.N = 2.0
+dirichlet.Geom.domain.Saturation.SRes = 0.2
+dirichlet.Geom.domain.Saturation.SSat = 1.0
+
+# -----------------------------------------------------------------------------
+# Wells
+# -----------------------------------------------------------------------------
+dirichlet.Wells.Names = ""
+
+
+# -----------------------------------------------------------------------------
+# Time Cycles
+# -----------------------------------------------------------------------------
+dirichlet.Cycle.Names = "constant"
+dirichlet.Cycle.constant.Names = "alltime"
+dirichlet.Cycle.constant.alltime.Length = 1
+dirichlet.Cycle.constant.Repeat = -1
+
+# -----------------------------------------------------------------------------
+# Boundary Conditions: Pressure
+# -----------------------------------------------------------------------------
+dirichlet.BCPressure.PatchNames = "left right front back bottom top"
+
+dirichlet.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+dirichlet.Patch.left.BCPressure.Cycle = "constant"
+dirichlet.Patch.left.BCPressure.RefGeom = "domain"
+dirichlet.Patch.left.BCPressure.RefPatch = "bottom"
+dirichlet.Patch.left.BCPressure.alltime.Value = 0.5
+
+dirichlet.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+dirichlet.Patch.right.BCPressure.Cycle = "constant"
+dirichlet.Patch.right.BCPressure.RefGeom = "domain"
+dirichlet.Patch.right.BCPressure.RefPatch = "bottom"
+dirichlet.Patch.right.BCPressure.alltime.Value = 0.5
+
+dirichlet.Patch.front.BCPressure.Type = "DirEquilRefPatch"
+dirichlet.Patch.front.BCPressure.Cycle = "constant"
+dirichlet.Patch.front.BCPressure.RefGeom = "domain"
+dirichlet.Patch.front.BCPressure.RefPatch = "bottom"
+dirichlet.Patch.front.BCPressure.alltime.Value = 0.5
+
+dirichlet.Patch.back.BCPressure.Type = "DirEquilRefPatch"
+dirichlet.Patch.back.BCPressure.Cycle = "constant"
+dirichlet.Patch.back.BCPressure.RefGeom = "domain"
+dirichlet.Patch.back.BCPressure.RefPatch = "bottom"
+dirichlet.Patch.back.BCPressure.alltime.Value = 0.5
+
+dirichlet.Patch.bottom.BCPressure.Type = "DirEquilRefPatch"
+dirichlet.Patch.bottom.BCPressure.Cycle = "constant"
+dirichlet.Patch.bottom.BCPressure.RefGeom = "domain"
+dirichlet.Patch.bottom.BCPressure.RefPatch = "bottom"
+dirichlet.Patch.bottom.BCPressure.alltime.Value = 0.5
+
+dirichlet.Patch.top.BCPressure.Type = "DirEquilRefPatch"
+dirichlet.Patch.top.BCPressure.Cycle = "constant"
+dirichlet.Patch.top.BCPressure.RefGeom = "domain"
+dirichlet.Patch.top.BCPressure.RefPatch = "top"
+dirichlet.Patch.top.BCPressure.alltime.Value = -0.5
+
+# ---------------------------------------------------------
+# Topo slopes in x-direction
+# ---------------------------------------------------------
+# topo slopes do not figure into the impes (fully sat) case but we still
+# need keys for them
+
+dirichlet.TopoSlopesX.Type = "Constant"
+dirichlet.TopoSlopesX.GeomNames = ""
+
+dirichlet.TopoSlopesX.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Topo slopes in y-direction
+# ---------------------------------------------------------
+
+dirichlet.TopoSlopesY.Type = "Constant"
+dirichlet.TopoSlopesY.GeomNames = ""
+
+dirichlet.TopoSlopesY.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
+# mannings roughnesses do not figure into the impes (fully sat) case but we still
+# need a key for them
+
+dirichlet.Mannings.Type = "Constant"
+dirichlet.Mannings.GeomNames = ""
+dirichlet.Mannings.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Initial conditions: water pressure
+# ---------------------------------------------------------
+
+dirichlet.ICPressure.Type = "HydroStaticPatch"
+dirichlet.ICPressure.GeomNames = "domain"
+dirichlet.Geom.domain.ICPressure.Value = 0.5
+dirichlet.Geom.domain.ICPressure.RefGeom = "domain"
+dirichlet.Geom.domain.ICPressure.RefPatch = "bottom"
+
+# -----------------------------------------------------------------------------
+# Phase sources:
+# -----------------------------------------------------------------------------
+
+dirichlet.PhaseSources.water.Type = "Constant"
+dirichlet.PhaseSources.water.GeomNames = "domain"
+dirichlet.PhaseSources.water.Geom.domain.Value = 0.0
+
+# -----------------------------------------------------------------------------
+# Exact solution specification for error calculations
+# -----------------------------------------------------------------------------
+
+dirichlet.KnownSolution = "NoKnownSolution"
+
+# -----------------------------------------------------------------------------
+# Set solver parameters
+# -----------------------------------------------------------------------------
+dirichlet.Solver = "Richards"
+dirichlet.Solver.MaxIter = 5
+
+dirichlet.Solver.Nonlinear.MaxIter = 10
+dirichlet.Solver.Nonlinear.ResidualTol = 1e-9
+dirichlet.Solver.Nonlinear.EtaChoice = "EtaConstant"
+dirichlet.Solver.Nonlinear.EtaValue = 1e-5
+dirichlet.Solver.Nonlinear.UseJacobian = True
+dirichlet.Solver.Nonlinear.DerivativeEpsilon = 1e-2
+
+dirichlet.Solver.Linear.KrylovDimension = 10
+
+dirichlet.Solver.Linear.Preconditioner = "PFMG"
+
+dirichlet.Solver.PrintVelocities = True
+
+
+# -----------------------------------------------------------------------------
+# Run and Unload the ParFlow output files
+# -----------------------------------------------------------------------------
+new_output_dir_name = get_absolute_path("test_output/dirichlet")
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+dirichlet.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+sig_digits = 6
+abs_value = 1e-12
+for i in range(0, 2):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.velx.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in x-velocity for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+    filename = f"/{run_name}.out.vely.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in y-velocity for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+    filename = f"/{run_name}.out.vely.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in z-velocity for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+rm(new_output_dir_name)
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/LW_surface_press.py b/test/python/LW_surface_press.py
index ac354abee..f0a8e7bcc 100644
--- a/test/python/LW_surface_press.py
+++ b/test/python/LW_surface_press.py
@@ -1,25 +1,27 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  This runs a Little Washita test problem with variable dz
 #  and adjusts surface pressures
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
+import sys
 from parflow import Run
-from parflow.tools.fs import cp, mkdir, chdir, get_absolute_path
+from parflow.tools.fs import cp, mkdir, chdir, get_absolute_path, rm
+from parflow.tools.compare import pf_test_file
 
-LW_surface_press = Run("LW_surface_press", __file__)
+run_name = "LW_surface_press"
 
-#---------------------------------------------------------
-# Copying slope files
-#---------------------------------------------------------
+LW_surface_press = Run(run_name, __file__)
 
-#dir_name = get_absolute_path('test_output/LW_surface_press')
-#mkdir(dir_name)
-#chdir(dir_name)
+# ---------------------------------------------------------
+# Copying slope files
+# ---------------------------------------------------------
+new_output_dir_name = get_absolute_path("test_output/LW_surface_press")
+mkdir(new_output_dir_name)
 
-#cp('../../test/input/lw.1km.slope_x.10x.pfb','.')
-#cp('../../test/input/lw.1km.slope_y.10x.pfb','.')
+cp("../../test/input/lw.1km.slope_x.10x.pfb", new_output_dir_name)
+cp("../../test/input/lw.1km.slope_y.10x.pfb", new_output_dir_name)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 LW_surface_press.FileVersion = 4
 
@@ -27,9 +29,9 @@
 LW_surface_press.Process.Topology.Q = 1
 LW_surface_press.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 LW_surface_press.ComputationalGrid.Lower.X = 0.0
 LW_surface_press.ComputationalGrid.Lower.Y = 0.0
@@ -43,22 +45,22 @@
 
 LW_surface_press.ComputationalGrid.DX = 1000.0
 LW_surface_press.ComputationalGrid.DY = 1000.0
-#"native" grid resolution is 2m everywhere X NZ=25 for 50m
-#computational domain.
+# "native" grid resolution is 2m everywhere X NZ=25 for 50m
+# computational domain.
 LW_surface_press.ComputationalGrid.DZ = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LW_surface_press.GeomInput.Names = 'domaininput'
+LW_surface_press.GeomInput.Names = "domaininput"
 
-LW_surface_press.GeomInput.domaininput.GeomName = 'domain'
-LW_surface_press.GeomInput.domaininput.InputType = 'Box'
+LW_surface_press.GeomInput.domaininput.GeomName = "domain"
+LW_surface_press.GeomInput.domaininput.InputType = "Box"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 LW_surface_press.Geom.domain.Lower.X = 0.0
 LW_surface_press.Geom.domain.Lower.Y = 0.0
@@ -68,15 +70,15 @@
 LW_surface_press.Geom.domain.Upper.Y = 32000.0
 # this upper is synched to computational grid, not linked w/ Z multipliers
 LW_surface_press.Geom.domain.Upper.Z = 12.0
-LW_surface_press.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+LW_surface_press.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#--------------------------------------------
+# --------------------------------------------
 # variable dz assignments
-#--------------------------------------------
+# --------------------------------------------
 
 LW_surface_press.Solver.Nonlinear.VariableDz = True
-LW_surface_press.dzScale.GeomNames = 'domain'
-LW_surface_press.dzScale.Type = 'nzList'
+LW_surface_press.dzScale.GeomNames = "domain"
+LW_surface_press.dzScale.Type = "nzList"
 LW_surface_press.dzScale.nzListNumber = 6
 
 LW_surface_press.Cell._0.dzScale.Value = 1.0
@@ -86,15 +88,15 @@
 LW_surface_press.Cell._4.dzScale.Value = 1.000
 LW_surface_press.Cell._5.dzScale.Value = 0.05
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_surface_press.Geom.Perm.Names = 'domain'
+LW_surface_press.Geom.Perm.Names = "domain"
 
 # Values in m/hour
 
-LW_surface_press.Geom.domain.Perm.Type = 'Constant'
+LW_surface_press.Geom.domain.Perm.Type = "Constant"
 LW_surface_press.Geom.domain.Perm.Value = 0.1
 
 LW_surface_press.Geom.domain.Perm.LambdaX = 5000.0
@@ -109,212 +111,212 @@
 LW_surface_press.Geom.domain.Perm.KMax = 100.0000001
 LW_surface_press.Geom.domain.Perm.DelK = 0.2
 LW_surface_press.Geom.domain.Perm.Seed = 33333
-LW_surface_press.Geom.domain.Perm.LogNormal = 'Log'
-LW_surface_press.Geom.domain.Perm.StratType = 'Bottom'
+LW_surface_press.Geom.domain.Perm.LogNormal = "Log"
+LW_surface_press.Geom.domain.Perm.StratType = "Bottom"
 
-LW_surface_press.Perm.TensorType = 'TensorByGeom'
+LW_surface_press.Perm.TensorType = "TensorByGeom"
 
-LW_surface_press.Geom.Perm.TensorByGeom.Names = 'domain'
+LW_surface_press.Geom.Perm.TensorByGeom.Names = "domain"
 
 LW_surface_press.Geom.domain.Perm.TensorValX = 1.0
 LW_surface_press.Geom.domain.Perm.TensorValY = 1.0
 LW_surface_press.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_surface_press.SpecificStorage.Type = 'Constant'
-LW_surface_press.SpecificStorage.GeomNames = 'domain'
+LW_surface_press.SpecificStorage.Type = "Constant"
+LW_surface_press.SpecificStorage.GeomNames = "domain"
 LW_surface_press.Geom.domain.SpecificStorage.Value = 1.0e-5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_surface_press.Phase.Names = 'water'
+LW_surface_press.Phase.Names = "water"
 
-LW_surface_press.Phase.water.Density.Type = 'Constant'
+LW_surface_press.Phase.water.Density.Type = "Constant"
 LW_surface_press.Phase.water.Density.Value = 1.0
 
-LW_surface_press.Phase.water.Viscosity.Type = 'Constant'
+LW_surface_press.Phase.water.Viscosity.Type = "Constant"
 LW_surface_press.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_surface_press.Contaminants.Names = ''
+LW_surface_press.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_surface_press.Geom.Retardation.GeomNames = ''
+LW_surface_press.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_surface_press.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_surface_press.TimingInfo.BaseUnit = 10.0
 LW_surface_press.TimingInfo.StartCount = 0
 LW_surface_press.TimingInfo.StartTime = 0.0
 LW_surface_press.TimingInfo.StopTime = 1000.0
 LW_surface_press.TimingInfo.DumpInterval = 100.0
-LW_surface_press.TimeStep.Type = 'Constant'
+LW_surface_press.TimeStep.Type = "Constant"
 LW_surface_press.TimeStep.Value = 100.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_surface_press.Geom.Porosity.GeomNames = 'domain'
-LW_surface_press.Geom.domain.Porosity.Type = 'Constant'
+LW_surface_press.Geom.Porosity.GeomNames = "domain"
+LW_surface_press.Geom.domain.Porosity.Type = "Constant"
 LW_surface_press.Geom.domain.Porosity.Value = 0.25
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_surface_press.Domain.GeomName = 'domain'
+LW_surface_press.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_surface_press.Phase.RelPerm.Type = 'VanGenuchten'
-LW_surface_press.Phase.RelPerm.GeomNames = 'domain'
+LW_surface_press.Phase.RelPerm.Type = "VanGenuchten"
+LW_surface_press.Phase.RelPerm.GeomNames = "domain"
 
-LW_surface_press.Geom.domain.RelPerm.Alpha = 1.
 LW_surface_press.Geom.domain.RelPerm.Alpha = 1.0
-LW_surface_press.Geom.domain.RelPerm.N = 3.
+LW_surface_press.Geom.domain.RelPerm.Alpha = 1.0
+LW_surface_press.Geom.domain.RelPerm.N = 3.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LW_surface_press.Phase.Saturation.Type = 'VanGenuchten'
-LW_surface_press.Phase.Saturation.GeomNames = 'domain'
+LW_surface_press.Phase.Saturation.Type = "VanGenuchten"
+LW_surface_press.Phase.Saturation.GeomNames = "domain"
 
 LW_surface_press.Geom.domain.Saturation.Alpha = 1.0
 LW_surface_press.Geom.domain.Saturation.Alpha = 1.0
-LW_surface_press.Geom.domain.Saturation.N = 3.
+LW_surface_press.Geom.domain.Saturation.N = 3.0
 LW_surface_press.Geom.domain.Saturation.SRes = 0.1
 LW_surface_press.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_surface_press.Wells.Names = ''
+LW_surface_press.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_surface_press.Cycle.Names = 'constant rainrec'
-LW_surface_press.Cycle.Names = 'constant'
-LW_surface_press.Cycle.constant.Names = 'alltime'
+LW_surface_press.Cycle.Names = "constant rainrec"
+LW_surface_press.Cycle.Names = "constant"
+LW_surface_press.Cycle.constant.Names = "alltime"
 LW_surface_press.Cycle.constant.alltime.Length = 10000000
 LW_surface_press.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-LW_surface_press.Cycle.rainrec.Names = 'rain rec'
+LW_surface_press.Cycle.rainrec.Names = "rain rec"
 LW_surface_press.Cycle.rainrec.rain.Length = 10
 LW_surface_press.Cycle.rainrec.rec.Length = 20
 LW_surface_press.Cycle.rainrec.Repeat = 14
 #
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_surface_press.BCPressure.PatchNames = LW_surface_press.Geom.domain.Patches
 
-LW_surface_press.Patch.x_lower.BCPressure.Type = 'FluxConst'
-LW_surface_press.Patch.x_lower.BCPressure.Cycle = 'constant'
+LW_surface_press.Patch.x_lower.BCPressure.Type = "FluxConst"
+LW_surface_press.Patch.x_lower.BCPressure.Cycle = "constant"
 LW_surface_press.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-LW_surface_press.Patch.y_lower.BCPressure.Type = 'FluxConst'
-LW_surface_press.Patch.y_lower.BCPressure.Cycle = 'constant'
+LW_surface_press.Patch.y_lower.BCPressure.Type = "FluxConst"
+LW_surface_press.Patch.y_lower.BCPressure.Cycle = "constant"
 LW_surface_press.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-LW_surface_press.Patch.z_lower.BCPressure.Type = 'FluxConst'
-LW_surface_press.Patch.z_lower.BCPressure.Cycle = 'constant'
+LW_surface_press.Patch.z_lower.BCPressure.Type = "FluxConst"
+LW_surface_press.Patch.z_lower.BCPressure.Cycle = "constant"
 LW_surface_press.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-LW_surface_press.Patch.x_upper.BCPressure.Type = 'FluxConst'
-LW_surface_press.Patch.x_upper.BCPressure.Cycle = 'constant'
+LW_surface_press.Patch.x_upper.BCPressure.Type = "FluxConst"
+LW_surface_press.Patch.x_upper.BCPressure.Cycle = "constant"
 LW_surface_press.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-LW_surface_press.Patch.y_upper.BCPressure.Type = 'FluxConst'
-LW_surface_press.Patch.y_upper.BCPressure.Cycle = 'constant'
+LW_surface_press.Patch.y_upper.BCPressure.Type = "FluxConst"
+LW_surface_press.Patch.y_upper.BCPressure.Cycle = "constant"
 LW_surface_press.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
 ## overland flow boundary condition with very heavy rainfall then slight ET
-LW_surface_press.Patch.z_upper.BCPressure.Type = 'FluxConst'
-LW_surface_press.Patch.z_upper.BCPressure.Cycle = 'constant'
+LW_surface_press.Patch.z_upper.BCPressure.Type = "FluxConst"
+LW_surface_press.Patch.z_upper.BCPressure.Cycle = "constant"
 # constant recharge at 100 mm / y
 LW_surface_press.Patch.z_upper.BCPressure.alltime.Value = -0.005
 LW_surface_press.Patch.z_upper.BCPressure.alltime.Value = -0.0001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LW_surface_press.TopoSlopesX.Type = 'PFBFile'
-LW_surface_press.TopoSlopesX.GeomNames = 'domain'
-LW_surface_press.TopoSlopesX.FileName = 'lw.1km.slope_x.10x.pfb'
+LW_surface_press.TopoSlopesX.Type = "PFBFile"
+LW_surface_press.TopoSlopesX.GeomNames = "domain"
+LW_surface_press.TopoSlopesX.FileName = "lw.1km.slope_x.10x.pfb"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LW_surface_press.TopoSlopesY.Type = 'PFBFile'
-LW_surface_press.TopoSlopesY.GeomNames = 'domain'
-LW_surface_press.TopoSlopesY.FileName = 'lw.1km.slope_y.10x.pfb'
+LW_surface_press.TopoSlopesY.Type = "PFBFile"
+LW_surface_press.TopoSlopesY.GeomNames = "domain"
+LW_surface_press.TopoSlopesY.FileName = "lw.1km.slope_y.10x.pfb"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  Distribute slopes
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LW_surface_press.dist('lw.1km.slope_x.10x.pfb')
-LW_surface_press.dist('lw.1km.slope_y.10x.pfb')
+LW_surface_press.dist(new_output_dir_name + "/lw.1km.slope_x.10x.pfb")
+LW_surface_press.dist(new_output_dir_name + "/lw.1km.slope_y.10x.pfb")
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LW_surface_press.Mannings.Type = 'Constant'
-LW_surface_press.Mannings.GeomNames = 'domain'
+LW_surface_press.Mannings.Type = "Constant"
+LW_surface_press.Mannings.GeomNames = "domain"
 LW_surface_press.Mannings.Geom.domain.Value = 0.00005
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_surface_press.PhaseSources.water.Type = 'Constant'
-LW_surface_press.PhaseSources.water.GeomNames = 'domain'
+LW_surface_press.PhaseSources.water.Type = "Constant"
+LW_surface_press.PhaseSources.water.GeomNames = "domain"
 LW_surface_press.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_surface_press.KnownSolution = 'NoKnownSolution'
+LW_surface_press.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_surface_press.Solver = 'Richards'
+LW_surface_press.Solver = "Richards"
 LW_surface_press.Solver.MaxIter = 2500
 
 LW_surface_press.Solver.TerrainFollowingGrid = True
@@ -324,38 +326,74 @@
 LW_surface_press.Solver.Nonlinear.EtaValue = 0.001
 
 LW_surface_press.Solver.PrintSubsurf = False
-LW_surface_press.Solver.Drop = 1E-20
-LW_surface_press.Solver.AbsTol = 1E-10
+LW_surface_press.Solver.Drop = 1e-20
+LW_surface_press.Solver.AbsTol = 1e-10
 
-LW_surface_press.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+LW_surface_press.Solver.Nonlinear.EtaChoice = "EtaConstant"
 LW_surface_press.Solver.Nonlinear.EtaValue = 0.001
 LW_surface_press.Solver.Nonlinear.UseJacobian = True
 LW_surface_press.Solver.Nonlinear.StepTol = 1e-25
-LW_surface_press.Solver.Nonlinear.Globalization = 'LineSearch'
+LW_surface_press.Solver.Nonlinear.Globalization = "LineSearch"
 LW_surface_press.Solver.Linear.KrylovDimension = 80
 LW_surface_press.Solver.Linear.MaxRestarts = 2
 
-LW_surface_press.Solver.Linear.Preconditioner = 'MGSemi'
+LW_surface_press.Solver.Linear.Preconditioner = "MGSemi"
 
 
 ##---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be at the bottom of the domain, the top layer is initially dry
-LW_surface_press.ICPressure.Type = 'HydroStaticPatch'
-LW_surface_press.ICPressure.GeomNames = 'domain'
+LW_surface_press.ICPressure.Type = "HydroStaticPatch"
+LW_surface_press.ICPressure.GeomNames = "domain"
 LW_surface_press.Geom.domain.ICPressure.Value = 0.0
 
-LW_surface_press.Geom.domain.ICPressure.RefGeom = 'domain'
-LW_surface_press.Geom.domain.ICPressure.RefPatch = 'z_upper'
+LW_surface_press.Geom.domain.ICPressure.RefGeom = "domain"
+LW_surface_press.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-LW_surface_press.Solver.ResetSurfacePressure  =  True
-LW_surface_press.Solver.ResetSurfacePressure.ThresholdPressure  = 10. 
-LW_surface_press.Solver.ResetSurfacePressure.ResetPressure  = -0.00001
+LW_surface_press.Solver.ResetSurfacePressure = True
+LW_surface_press.Solver.ResetSurfacePressure.ThresholdPressure = 10.0
+LW_surface_press.Solver.ResetSurfacePressure.ResetPressure = -0.00001
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and do tests
-#-----------------------------------------------------------------------------
-
-LW_surface_press.run()
+# -----------------------------------------------------------------------------
+correct_output_dir_name = get_absolute_path("../correct_output")
+LW_surface_press.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+for i in range(0, 11):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+rm(new_output_dir_name)
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/LW_var_dz.py b/test/python/LW_var_dz.py
index afa4bb2aa..0cd781cf3 100644
--- a/test/python/LW_var_dz.py
+++ b/test/python/LW_var_dz.py
@@ -1,36 +1,44 @@
-#--------------------------------------------------------------
+# --------------------------------------------------------------
 #  This runs the a Little Washita Test Problem with variable dz
 #  and a constant rain forcing.  The full Jacobian is use and there is no dampening in the
 #  overland flow.
-#--------------------------------------------------------------
-
+# --------------------------------------------------------------
+import os
+import sys, argparse
 from parflow import Run
-from parflow.tools.fs import cp, mkdir, chdir, get_absolute_path
+from parflow.tools.fs import cp, mkdir, chdir, get_absolute_path, rm
+from parflow.tools.compare import pf_test_file, pf_test_file_with_abs
 
-LWvdz = Run("LWvdz", __file__)
+run_name = "LW_var_dz"
+LWvdz = Run(run_name, __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Copying slope files
-#---------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/LWvdz')
-mkdir(dir_name)
-chdir(dir_name)
+# ---------------------------------------------------------
+new_output_dir_name = get_absolute_path("test_output/LW_var_dz")
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
 
-cp('$PF_SRC/test/input/lw.1km.slope_x.10x.pfb')
-cp('$PF_SRC/test/input/lw.1km.slope_y.10x.pfb')
+cp("$PF_SRC/test/input/lw.1km.slope_x.10x.pfb", new_output_dir_name)
+cp("$PF_SRC/test/input/lw.1km.slope_y.10x.pfb", new_output_dir_name)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 LWvdz.FileVersion = 4
 
-LWvdz.Process.Topology.P = 1
-LWvdz.Process.Topology.Q = 1
-LWvdz.Process.Topology.R = 1
+parser = argparse.ArgumentParser()
+parser.add_argument("-p", "--p", default=1)
+parser.add_argument("-q", "--q", default=1)
+parser.add_argument("-r", "--r", default=1)
+args = parser.parse_args()
 
-#---------------------------------------------------------
+LWvdz.Process.Topology.P = args.p
+LWvdz.Process.Topology.Q = args.q
+LWvdz.Process.Topology.R = args.r
+
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 LWvdz.ComputationalGrid.Lower.X = 0.0
 LWvdz.ComputationalGrid.Lower.Y = 0.0
@@ -44,22 +52,22 @@
 
 LWvdz.ComputationalGrid.DX = 1000.0
 LWvdz.ComputationalGrid.DY = 1000.0
-#"native" grid resolution is 2m everywhere X NZ=25 for 50m
-#computational domain.
+# "native" grid resolution is 2m everywhere X NZ=25 for 50m
+# computational domain.
 LWvdz.ComputationalGrid.DZ = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LWvdz.GeomInput.Names = 'domaininput'
+LWvdz.GeomInput.Names = "domaininput"
 
-LWvdz.GeomInput.domaininput.GeomName = 'domain'
-LWvdz.GeomInput.domaininput.InputType = 'Box'
+LWvdz.GeomInput.domaininput.GeomName = "domain"
+LWvdz.GeomInput.domaininput.InputType = "Box"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 LWvdz.Geom.domain.Lower.X = 0.0
 LWvdz.Geom.domain.Lower.Y = 0.0
@@ -69,15 +77,15 @@
 LWvdz.Geom.domain.Upper.Y = 32000.0
 # this upper is synched to computational grid, not linked w/ Z multipliers
 LWvdz.Geom.domain.Upper.Z = 12.0
-LWvdz.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+LWvdz.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#--------------------------------------------
+# --------------------------------------------
 # variable dz assignments
-#--------------------------------------------
+# --------------------------------------------
 
 LWvdz.Solver.Nonlinear.VariableDz = True
-LWvdz.dzScale.GeomNames = 'domain'
-LWvdz.dzScale.Type = 'nzList'
+LWvdz.dzScale.GeomNames = "domain"
+LWvdz.dzScale.Type = "nzList"
 LWvdz.dzScale.nzListNumber = 6
 
 LWvdz.Cell._0.dzScale.Value = 1.0
@@ -87,17 +95,17 @@
 LWvdz.Cell._4.dzScale.Value = 1.000
 LWvdz.Cell._5.dzScale.Value = 0.05
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Geom.Perm.Names = 'domain'
+LWvdz.Geom.Perm.Names = "domain"
 
 # Values in m/hour
 
-LWvdz.Geom.domain.Perm.Type = 'Constant'
+LWvdz.Geom.domain.Perm.Type = "Constant"
 
-LWvdz.Geom.domain.Perm.Type = 'TurnBands'
+LWvdz.Geom.domain.Perm.Type = "TurnBands"
 LWvdz.Geom.domain.Perm.LambdaX = 5000.0
 LWvdz.Geom.domain.Perm.LambdaY = 5000.0
 LWvdz.Geom.domain.Perm.LambdaZ = 50.0
@@ -110,210 +118,209 @@
 LWvdz.Geom.domain.Perm.KMax = 100.0000001
 LWvdz.Geom.domain.Perm.DelK = 0.2
 LWvdz.Geom.domain.Perm.Seed = 33333
-LWvdz.Geom.domain.Perm.LogNormal = 'Log'
-LWvdz.Geom.domain.Perm.StratType = 'Bottom'
+LWvdz.Geom.domain.Perm.LogNormal = "Log"
+LWvdz.Geom.domain.Perm.StratType = "Bottom"
 
-LWvdz.Perm.TensorType = 'TensorByGeom'
+LWvdz.Perm.TensorType = "TensorByGeom"
 
-LWvdz.Geom.Perm.TensorByGeom.Names = 'domain'
+LWvdz.Geom.Perm.TensorByGeom.Names = "domain"
 
 LWvdz.Geom.domain.Perm.TensorValX = 1.0
 LWvdz.Geom.domain.Perm.TensorValY = 1.0
 LWvdz.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.SpecificStorage.Type = 'Constant'
-LWvdz.SpecificStorage.GeomNames = 'domain'
+LWvdz.SpecificStorage.Type = "Constant"
+LWvdz.SpecificStorage.GeomNames = "domain"
 LWvdz.Geom.domain.SpecificStorage.Value = 1.0e-5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Phase.Names = 'water'
+LWvdz.Phase.Names = "water"
 
-LWvdz.Phase.water.Density.Type = 'Constant'
+LWvdz.Phase.water.Density.Type = "Constant"
 LWvdz.Phase.water.Density.Value = 1.0
 
-LWvdz.Phase.water.Viscosity.Type = 'Constant'
+LWvdz.Phase.water.Viscosity.Type = "Constant"
 LWvdz.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Contaminants.Names = ''
+LWvdz.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Geom.Retardation.GeomNames = ''
+LWvdz.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LWvdz.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LWvdz.TimingInfo.BaseUnit = 10.0
 LWvdz.TimingInfo.StartCount = 0
 LWvdz.TimingInfo.StartTime = 0.0
 LWvdz.TimingInfo.StopTime = 200.0
 LWvdz.TimingInfo.DumpInterval = 20.0
-LWvdz.TimeStep.Type = 'Constant'
+LWvdz.TimeStep.Type = "Constant"
 LWvdz.TimeStep.Value = 10.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Geom.Porosity.GeomNames = 'domain'
-LWvdz.Geom.domain.Porosity.Type = 'Constant'
+LWvdz.Geom.Porosity.GeomNames = "domain"
+LWvdz.Geom.domain.Porosity.Type = "Constant"
 LWvdz.Geom.domain.Porosity.Value = 0.25
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Domain.GeomName = 'domain'
+LWvdz.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Phase.RelPerm.Type = 'VanGenuchten'
-LWvdz.Phase.RelPerm.GeomNames = 'domain'
+LWvdz.Phase.RelPerm.Type = "VanGenuchten"
+LWvdz.Phase.RelPerm.GeomNames = "domain"
 
-LWvdz.Geom.domain.RelPerm.Alpha = 1.
 LWvdz.Geom.domain.RelPerm.Alpha = 1.0
-LWvdz.Geom.domain.RelPerm.N = 3.
+LWvdz.Geom.domain.RelPerm.Alpha = 1.0
+LWvdz.Geom.domain.RelPerm.N = 3.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LWvdz.Phase.Saturation.Type = 'VanGenuchten'
-LWvdz.Phase.Saturation.GeomNames = 'domain'
+LWvdz.Phase.Saturation.Type = "VanGenuchten"
+LWvdz.Phase.Saturation.GeomNames = "domain"
 
 LWvdz.Geom.domain.Saturation.Alpha = 1.0
 LWvdz.Geom.domain.Saturation.Alpha = 1.0
-LWvdz.Geom.domain.Saturation.N = 3.
+LWvdz.Geom.domain.Saturation.N = 3.0
 LWvdz.Geom.domain.Saturation.SRes = 0.1
 LWvdz.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Wells.Names = ''
+LWvdz.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Cycle.Names = 'constant rainrec'
-LWvdz.Cycle.constant.Names = 'alltime'
+LWvdz.Cycle.Names = "constant rainrec"
+LWvdz.Cycle.constant.Names = "alltime"
 LWvdz.Cycle.constant.alltime.Length = 10000000
 LWvdz.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-LWvdz.Cycle.rainrec.Names = 'rain rec'
+LWvdz.Cycle.rainrec.Names = "rain rec"
 LWvdz.Cycle.rainrec.rain.Length = 10
 LWvdz.Cycle.rainrec.rec.Length = 20
 LWvdz.Cycle.rainrec.Repeat = 14
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LWvdz.BCPressure.PatchNames = LWvdz.Geom.domain.Patches
 
-LWvdz.Patch.x_lower.BCPressure.Type = 'FluxConst'
-LWvdz.Patch.x_lower.BCPressure.Cycle = 'constant'
+LWvdz.Patch.x_lower.BCPressure.Type = "FluxConst"
+LWvdz.Patch.x_lower.BCPressure.Cycle = "constant"
 LWvdz.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-LWvdz.Patch.y_lower.BCPressure.Type = 'FluxConst'
-LWvdz.Patch.y_lower.BCPressure.Cycle = 'constant'
+LWvdz.Patch.y_lower.BCPressure.Type = "FluxConst"
+LWvdz.Patch.y_lower.BCPressure.Cycle = "constant"
 LWvdz.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-LWvdz.Patch.z_lower.BCPressure.Type = 'FluxConst'
-LWvdz.Patch.z_lower.BCPressure.Cycle = 'constant'
+LWvdz.Patch.z_lower.BCPressure.Type = "FluxConst"
+LWvdz.Patch.z_lower.BCPressure.Cycle = "constant"
 LWvdz.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-LWvdz.Patch.x_upper.BCPressure.Type = 'FluxConst'
-LWvdz.Patch.x_upper.BCPressure.Cycle = 'constant'
+LWvdz.Patch.x_upper.BCPressure.Type = "FluxConst"
+LWvdz.Patch.x_upper.BCPressure.Cycle = "constant"
 LWvdz.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-LWvdz.Patch.y_upper.BCPressure.Type = 'FluxConst'
-LWvdz.Patch.y_upper.BCPressure.Cycle = 'constant'
+LWvdz.Patch.y_upper.BCPressure.Type = "FluxConst"
+LWvdz.Patch.y_upper.BCPressure.Cycle = "constant"
 LWvdz.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
 ## overland flow boundary condition with very heavy rainfall
-LWvdz.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-LWvdz.Patch.z_upper.BCPressure.Cycle = 'constant'
+LWvdz.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+LWvdz.Patch.z_upper.BCPressure.Cycle = "constant"
 # constant recharge at 100 mm / y
 LWvdz.Patch.z_upper.BCPressure.alltime.Value = -0.005
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LWvdz.TopoSlopesX.Type = 'PFBFile'
-LWvdz.TopoSlopesX.GeomNames = 'domain'
-LWvdz.TopoSlopesX.FileName = 'lw.1km.slope_x.10x.pfb'
+LWvdz.TopoSlopesX.Type = "PFBFile"
+LWvdz.TopoSlopesX.GeomNames = "domain"
+LWvdz.TopoSlopesX.FileName = "lw.1km.slope_x.10x.pfb"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LWvdz.TopoSlopesY.Type = 'PFBFile'
-LWvdz.TopoSlopesY.GeomNames = 'domain'
-LWvdz.TopoSlopesY.FileName = 'lw.1km.slope_y.10x.pfb'
+LWvdz.TopoSlopesY.Type = "PFBFile"
+LWvdz.TopoSlopesY.GeomNames = "domain"
+LWvdz.TopoSlopesY.FileName = "lw.1km.slope_y.10x.pfb"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  Distribute slopes
-#---------------------------------------------------------
+# ---------------------------------------------------------
+LWvdz.dist(new_output_dir_name + "/lw.1km.slope_x.10x.pfb")
+LWvdz.dist(new_output_dir_name + "/lw.1km.slope_y.10x.pfb")
 
-LWvdz.dist('lw.1km.slope_x.10x.pfb')
-LWvdz.dist('lw.1km.slope_y.10x.pfb')
-
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LWvdz.Mannings.Type = 'Constant'
-LWvdz.Mannings.GeomNames = 'domain'
+LWvdz.Mannings.Type = "Constant"
+LWvdz.Mannings.GeomNames = "domain"
 LWvdz.Mannings.Geom.domain.Value = 0.00005
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.PhaseSources.water.Type = 'Constant'
-LWvdz.PhaseSources.water.GeomNames = 'domain'
+LWvdz.PhaseSources.water.Type = "Constant"
+LWvdz.PhaseSources.water.GeomNames = "domain"
 LWvdz.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.KnownSolution = 'NoKnownSolution'
+LWvdz.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Solver = 'Richards'
+LWvdz.Solver = "Richards"
 LWvdz.Solver.MaxIter = 2500
 
 LWvdz.Solver.TerrainFollowingGrid = True
@@ -323,40 +330,106 @@
 LWvdz.Solver.Nonlinear.EtaValue = 0.001
 
 LWvdz.Solver.PrintSubsurf = False
-LWvdz.Solver.Drop = 1E-20
-LWvdz.Solver.AbsTol = 1E-10
+LWvdz.Solver.Drop = 1e-20
+LWvdz.Solver.AbsTol = 1e-10
 
-LWvdz.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+LWvdz.Solver.Nonlinear.EtaChoice = "EtaConstant"
 LWvdz.Solver.Nonlinear.EtaValue = 0.001
 LWvdz.Solver.Nonlinear.UseJacobian = True
 
 LWvdz.Solver.Nonlinear.DerivativeEpsilon = 1e-14
 LWvdz.Solver.Nonlinear.StepTol = 1e-25
-LWvdz.Solver.Nonlinear.Globalization = 'LineSearch'
+LWvdz.Solver.Nonlinear.Globalization = "LineSearch"
 LWvdz.Solver.Linear.KrylovDimension = 80
 LWvdz.Solver.Linear.MaxRestarts = 2
 
-LWvdz.Solver.Linear.Preconditioner = 'MGSemi'
-LWvdz.Solver.Linear.Preconditioner = 'PFMG'
-LWvdz.Solver.Linear.Preconditioner.PCMatrixType = 'FullJacobian'
+LWvdz.Solver.Linear.Preconditioner = "MGSemi"
+LWvdz.Solver.Linear.Preconditioner = "PFMG"
+LWvdz.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
+
 
-#---------------------------------------------------------
+LWvdz.Solver.PrintVelocities = True
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be at the bottom of the domain, the top layer is initially dry
-LWvdz.ICPressure.Type = 'HydroStaticPatch'
-LWvdz.ICPressure.GeomNames = 'domain'
+LWvdz.ICPressure.Type = "HydroStaticPatch"
+LWvdz.ICPressure.GeomNames = "domain"
 LWvdz.Geom.domain.ICPressure.Value = -10.0
 
-LWvdz.Geom.domain.ICPressure.RefGeom = 'domain'
-LWvdz.Geom.domain.ICPressure.RefPatch = 'z_upper'
+LWvdz.Geom.domain.ICPressure.RefGeom = "domain"
+LWvdz.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#spinup key
+# spinup key
 LWvdz.Solver.Spinup = False
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-LWvdz.run()
+# -----------------------------------------------------------------------------
+LWvdz.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+sig_digits = 6
+abs_value = 1e-12
+for i in range(0, 12, 2):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.velx.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in x-velocity for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+    filename = f"/{run_name}.out.vely.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in y-velocity for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+    filename = f"/{run_name}.out.vely.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in z-velocity for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+rm(new_output_dir_name)
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/LW_var_dz_spinup.py b/test/python/LW_var_dz_spinup.py
index ba8e1a9d1..de5396f4c 100644
--- a/test/python/LW_var_dz_spinup.py
+++ b/test/python/LW_var_dz_spinup.py
@@ -1,35 +1,44 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  This runs a Little Washita test problem with variable dz
 #  in spinup mode with overland dampening turned on
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
+import sys, argparse
 from parflow import Run
-from parflow.tools.fs import cp, mkdir, chdir, get_absolute_path
+from parflow.tools.fs import cp, mkdir, chdir, get_absolute_path, rm
+from parflow.tools.compare import pf_test_file
 
-LWvdz = Run("LW_var_dz_spinup", __file__)
+run_name = "LW_var_dz_spinup"
+LWvdz = Run(run_name, __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Copying slope files
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/LWvdz_s')
-mkdir(dir_name)
-chdir(dir_name)
+new_output_dir_name = get_absolute_path("test_output/LW_var_dz_spinup")
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
 
-cp('$PF_SRC/test/input/lw.1km.slope_x.10x.pfb')
-cp('$PF_SRC/test/input/lw.1km.slope_y.10x.pfb')
+cp("$PF_SRC/test/input/lw.1km.slope_x.10x.pfb", new_output_dir_name)
+cp("$PF_SRC/test/input/lw.1km.slope_y.10x.pfb", new_output_dir_name)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 LWvdz.FileVersion = 4
 
-LWvdz.Process.Topology.P = 1
-LWvdz.Process.Topology.Q = 1
-LWvdz.Process.Topology.R = 1
+parser = argparse.ArgumentParser()
+parser.add_argument("-p", "--p", default=1)
+parser.add_argument("-q", "--q", default=1)
+parser.add_argument("-r", "--r", default=1)
+args = parser.parse_args()
 
-#---------------------------------------------------------
+LWvdz.Process.Topology.P = args.p
+LWvdz.Process.Topology.Q = args.q
+LWvdz.Process.Topology.R = args.r
+
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 LWvdz.ComputationalGrid.Lower.X = 0.0
 LWvdz.ComputationalGrid.Lower.Y = 0.0
@@ -43,22 +52,22 @@
 
 LWvdz.ComputationalGrid.DX = 1000.0
 LWvdz.ComputationalGrid.DY = 1000.0
-#"native" grid resolution is 2m everywhere X NZ=25 for 50m
-#computational domain.
+# "native" grid resolution is 2m everywhere X NZ=25 for 50m
+# computational domain.
 LWvdz.ComputationalGrid.DZ = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LWvdz.GeomInput.Names = 'domaininput'
+LWvdz.GeomInput.Names = "domaininput"
 
-LWvdz.GeomInput.domaininput.GeomName = 'domain'
-LWvdz.GeomInput.domaininput.InputType = 'Box'
+LWvdz.GeomInput.domaininput.GeomName = "domain"
+LWvdz.GeomInput.domaininput.InputType = "Box"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 LWvdz.Geom.domain.Lower.X = 0.0
 LWvdz.Geom.domain.Lower.Y = 0.0
@@ -68,15 +77,15 @@
 LWvdz.Geom.domain.Upper.Y = 32000.0
 # this upper is synched to computational grid, not linked w/ Z multipliers
 LWvdz.Geom.domain.Upper.Z = 12.0
-LWvdz.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+LWvdz.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#--------------------------------------------
+# --------------------------------------------
 # variable dz assignments
-#--------------------------------------------
+# --------------------------------------------
 
 LWvdz.Solver.Nonlinear.VariableDz = True
-LWvdz.dzScale.GeomNames = 'domain'
-LWvdz.dzScale.Type = 'nzList'
+LWvdz.dzScale.GeomNames = "domain"
+LWvdz.dzScale.Type = "nzList"
 LWvdz.dzScale.nzListNumber = 6
 
 LWvdz.Cell._0.dzScale.Value = 1.0
@@ -86,15 +95,15 @@
 LWvdz.Cell._4.dzScale.Value = 1.000
 LWvdz.Cell._5.dzScale.Value = 0.05
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Geom.Perm.Names = 'domain'
+LWvdz.Geom.Perm.Names = "domain"
 
 # Values in m/hour
 
-LWvdz.Geom.domain.Perm.Type = 'Constant'
+LWvdz.Geom.domain.Perm.Type = "Constant"
 LWvdz.Geom.domain.Perm.Value = 0.1
 
 LWvdz.Geom.domain.Perm.LambdaX = 5000.0
@@ -109,212 +118,212 @@
 LWvdz.Geom.domain.Perm.KMax = 100.0000001
 LWvdz.Geom.domain.Perm.DelK = 0.2
 LWvdz.Geom.domain.Perm.Seed = 33333
-LWvdz.Geom.domain.Perm.LogNormal = 'Log'
-LWvdz.Geom.domain.Perm.StratType = 'Bottom'
+LWvdz.Geom.domain.Perm.LogNormal = "Log"
+LWvdz.Geom.domain.Perm.StratType = "Bottom"
 
-LWvdz.Perm.TensorType = 'TensorByGeom'
+LWvdz.Perm.TensorType = "TensorByGeom"
 
-LWvdz.Geom.Perm.TensorByGeom.Names = 'domain'
+LWvdz.Geom.Perm.TensorByGeom.Names = "domain"
 
 LWvdz.Geom.domain.Perm.TensorValX = 1.0
 LWvdz.Geom.domain.Perm.TensorValY = 1.0
 LWvdz.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.SpecificStorage.Type = 'Constant'
-LWvdz.SpecificStorage.GeomNames = 'domain'
+LWvdz.SpecificStorage.Type = "Constant"
+LWvdz.SpecificStorage.GeomNames = "domain"
 LWvdz.Geom.domain.SpecificStorage.Value = 1.0e-5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Phase.Names = 'water'
+LWvdz.Phase.Names = "water"
 
-LWvdz.Phase.water.Density.Type = 'Constant'
+LWvdz.Phase.water.Density.Type = "Constant"
 LWvdz.Phase.water.Density.Value = 1.0
 
-LWvdz.Phase.water.Viscosity.Type = 'Constant'
+LWvdz.Phase.water.Viscosity.Type = "Constant"
 LWvdz.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Contaminants.Names = ''
+LWvdz.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Geom.Retardation.GeomNames = ''
+LWvdz.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LWvdz.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LWvdz.TimingInfo.BaseUnit = 10.0
 LWvdz.TimingInfo.StartCount = 0
 LWvdz.TimingInfo.StartTime = 0.0
 LWvdz.TimingInfo.StopTime = 7000.0
 LWvdz.TimingInfo.DumpInterval = 1000.0
-LWvdz.TimeStep.Type = 'Constant'
+LWvdz.TimeStep.Type = "Constant"
 LWvdz.TimeStep.Value = 1000.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Geom.Porosity.GeomNames = 'domain'
-LWvdz.Geom.domain.Porosity.Type = 'Constant'
+LWvdz.Geom.Porosity.GeomNames = "domain"
+LWvdz.Geom.domain.Porosity.Type = "Constant"
 LWvdz.Geom.domain.Porosity.Value = 0.25
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Domain.GeomName = 'domain'
+LWvdz.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Phase.RelPerm.Type = 'VanGenuchten'
-LWvdz.Phase.RelPerm.GeomNames = 'domain'
+LWvdz.Phase.RelPerm.Type = "VanGenuchten"
+LWvdz.Phase.RelPerm.GeomNames = "domain"
 
-LWvdz.Geom.domain.RelPerm.Alpha = 1.
 LWvdz.Geom.domain.RelPerm.Alpha = 1.0
-LWvdz.Geom.domain.RelPerm.N = 3.
+LWvdz.Geom.domain.RelPerm.Alpha = 1.0
+LWvdz.Geom.domain.RelPerm.N = 3.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LWvdz.Phase.Saturation.Type = 'VanGenuchten'
-LWvdz.Phase.Saturation.GeomNames = 'domain'
+LWvdz.Phase.Saturation.Type = "VanGenuchten"
+LWvdz.Phase.Saturation.GeomNames = "domain"
 
 LWvdz.Geom.domain.Saturation.Alpha = 1.0
 LWvdz.Geom.domain.Saturation.Alpha = 1.0
-LWvdz.Geom.domain.Saturation.N = 3.
+LWvdz.Geom.domain.Saturation.N = 3.0
 LWvdz.Geom.domain.Saturation.SRes = 0.1
 LWvdz.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Wells.Names = ''
+LWvdz.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Cycle.Names = 'constant rainrec'
-LWvdz.Cycle.Names = 'constant'
-LWvdz.Cycle.constant.Names = 'alltime'
+LWvdz.Cycle.Names = "constant rainrec"
+LWvdz.Cycle.Names = "constant"
+LWvdz.Cycle.constant.Names = "alltime"
 LWvdz.Cycle.constant.alltime.Length = 10000000
 LWvdz.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-LWvdz.Cycle.rainrec.Names = 'rain rec'
+LWvdz.Cycle.rainrec.Names = "rain rec"
 LWvdz.Cycle.rainrec.rain.Length = 10
 LWvdz.Cycle.rainrec.rec.Length = 20
 LWvdz.Cycle.rainrec.Repeat = 14
 #
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LWvdz.BCPressure.PatchNames = LWvdz.Geom.domain.Patches
 
-LWvdz.Patch.x_lower.BCPressure.Type = 'FluxConst'
-LWvdz.Patch.x_lower.BCPressure.Cycle = 'constant'
+LWvdz.Patch.x_lower.BCPressure.Type = "FluxConst"
+LWvdz.Patch.x_lower.BCPressure.Cycle = "constant"
 LWvdz.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-LWvdz.Patch.y_lower.BCPressure.Type = 'FluxConst'
-LWvdz.Patch.y_lower.BCPressure.Cycle = 'constant'
+LWvdz.Patch.y_lower.BCPressure.Type = "FluxConst"
+LWvdz.Patch.y_lower.BCPressure.Cycle = "constant"
 LWvdz.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-LWvdz.Patch.z_lower.BCPressure.Type = 'FluxConst'
-LWvdz.Patch.z_lower.BCPressure.Cycle = 'constant'
+LWvdz.Patch.z_lower.BCPressure.Type = "FluxConst"
+LWvdz.Patch.z_lower.BCPressure.Cycle = "constant"
 LWvdz.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-LWvdz.Patch.x_upper.BCPressure.Type = 'FluxConst'
-LWvdz.Patch.x_upper.BCPressure.Cycle = 'constant'
+LWvdz.Patch.x_upper.BCPressure.Type = "FluxConst"
+LWvdz.Patch.x_upper.BCPressure.Cycle = "constant"
 LWvdz.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-LWvdz.Patch.y_upper.BCPressure.Type = 'FluxConst'
-LWvdz.Patch.y_upper.BCPressure.Cycle = 'constant'
+LWvdz.Patch.y_upper.BCPressure.Type = "FluxConst"
+LWvdz.Patch.y_upper.BCPressure.Cycle = "constant"
 LWvdz.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
 ## overland flow boundary condition with very heavy rainfall then slight ET
-LWvdz.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-LWvdz.Patch.z_upper.BCPressure.Cycle = 'constant'
+LWvdz.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+LWvdz.Patch.z_upper.BCPressure.Cycle = "constant"
 # constant recharge at 100 mm / y
 LWvdz.Patch.z_upper.BCPressure.alltime.Value = -0.005
 LWvdz.Patch.z_upper.BCPressure.alltime.Value = -0.0001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LWvdz.TopoSlopesX.Type = 'PFBFile'
-LWvdz.TopoSlopesX.GeomNames = 'domain'
-LWvdz.TopoSlopesX.FileName = 'lw.1km.slope_x.10x.pfb'
+LWvdz.TopoSlopesX.Type = "PFBFile"
+LWvdz.TopoSlopesX.GeomNames = "domain"
+LWvdz.TopoSlopesX.FileName = "lw.1km.slope_x.10x.pfb"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LWvdz.TopoSlopesY.Type = 'PFBFile'
-LWvdz.TopoSlopesY.GeomNames = 'domain'
-LWvdz.TopoSlopesY.FileName = 'lw.1km.slope_y.10x.pfb'
+LWvdz.TopoSlopesY.Type = "PFBFile"
+LWvdz.TopoSlopesY.GeomNames = "domain"
+LWvdz.TopoSlopesY.FileName = "lw.1km.slope_y.10x.pfb"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  Distribute slopes
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LWvdz.dist('lw.1km.slope_x.10x.pfb')
-LWvdz.dist('lw.1km.slope_y.10x.pfb')
+LWvdz.dist(new_output_dir_name + "/lw.1km.slope_x.10x.pfb")
+LWvdz.dist(new_output_dir_name + "/lw.1km.slope_y.10x.pfb")
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LWvdz.Mannings.Type = 'Constant'
-LWvdz.Mannings.GeomNames = 'domain'
+LWvdz.Mannings.Type = "Constant"
+LWvdz.Mannings.GeomNames = "domain"
 LWvdz.Mannings.Geom.domain.Value = 0.00005
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.PhaseSources.water.Type = 'Constant'
-LWvdz.PhaseSources.water.GeomNames = 'domain'
+LWvdz.PhaseSources.water.Type = "Constant"
+LWvdz.PhaseSources.water.GeomNames = "domain"
 LWvdz.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.KnownSolution = 'NoKnownSolution'
+LWvdz.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LWvdz.Solver = 'Richards'
+LWvdz.Solver = "Richards"
 LWvdz.Solver.MaxIter = 2500
 
 LWvdz.Solver.TerrainFollowingGrid = True
@@ -324,40 +333,76 @@
 LWvdz.Solver.Nonlinear.EtaValue = 0.001
 
 LWvdz.Solver.PrintSubsurf = False
-LWvdz.Solver.Drop = 1E-20
-LWvdz.Solver.AbsTol = 1E-10
+LWvdz.Solver.Drop = 1e-20
+LWvdz.Solver.AbsTol = 1e-10
 
-LWvdz.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+LWvdz.Solver.Nonlinear.EtaChoice = "EtaConstant"
 LWvdz.Solver.Nonlinear.EtaValue = 0.001
 LWvdz.Solver.Nonlinear.UseJacobian = True
 LWvdz.Solver.Nonlinear.StepTol = 1e-25
-LWvdz.Solver.Nonlinear.Globalization = 'LineSearch'
+LWvdz.Solver.Nonlinear.Globalization = "LineSearch"
 LWvdz.Solver.Linear.KrylovDimension = 80
 LWvdz.Solver.Linear.MaxRestarts = 2
 
-LWvdz.Solver.Linear.Preconditioner = 'MGSemi'
-LWvdz.Solver.Linear.Preconditioner = 'PFMG'
-LWvdz.Solver.Linear.Preconditioner.PCMatrixType = 'FullJacobian'
+LWvdz.Solver.Linear.Preconditioner = "MGSemi"
+LWvdz.Solver.Linear.Preconditioner = "PFMG"
+LWvdz.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
 
 ##---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be at the bottom of the domain, the top layer is initially dry
-LWvdz.ICPressure.Type = 'HydroStaticPatch'
-LWvdz.ICPressure.GeomNames = 'domain'
+LWvdz.ICPressure.Type = "HydroStaticPatch"
+LWvdz.ICPressure.GeomNames = "domain"
 LWvdz.Geom.domain.ICPressure.Value = 0.0
 
-LWvdz.Geom.domain.ICPressure.RefGeom = 'domain'
-LWvdz.Geom.domain.ICPressure.RefPatch = 'z_upper'
+LWvdz.Geom.domain.ICPressure.RefGeom = "domain"
+LWvdz.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#spinup key
+# spinup key
 LWvdz.OverlandFlowSpinUp = 1
 LWvdz.OverlandSpinupDampP1 = 1.0
 LWvdz.OverlandSpinupDampP2 = 0.00001
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and do tests
-#-----------------------------------------------------------------------------
-
-LWvdz.run()
+# -----------------------------------------------------------------------------
+LWvdz.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+
+for i in range(8):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+rm(new_output_dir_name)
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/bc_flux_file.py b/test/python/bc_flux_file.py
new file mode 100644
index 000000000..a20437f29
--- /dev/null
+++ b/test/python/bc_flux_file.py
@@ -0,0 +1,431 @@
+# Test for using the "PressureFile" option for BC.
+# Test is based on default_richards with modified BC
+# specification.
+
+import sys
+from parflow import Run
+from parflow.tools.fs import mkdir, get_absolute_path, rm
+from parflow.tools.compare import pf_test_file, pf_test_file_with_abs
+from parflow.tools.io import write_pfb
+import numpy as np
+
+run_name = "bc_flux_file"
+bcf = Run(run_name, __file__)
+
+bcf.FileVersion = 4
+
+new_output_dir_name = get_absolute_path("test_output/bcf")
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+
+bcf.Process.Topology.P = 1
+bcf.Process.Topology.Q = 1
+bcf.Process.Topology.R = 1
+
+# ---------------------------------------------------------
+# Computational Grid
+# ---------------------------------------------------------
+bcf.ComputationalGrid.Lower.X = -10.0
+bcf.ComputationalGrid.Lower.Y = 10.0
+bcf.ComputationalGrid.Lower.Z = 1.0
+
+bcf.ComputationalGrid.DX = 8.8888888888888893
+bcf.ComputationalGrid.DY = 10.666666666666666
+bcf.ComputationalGrid.DZ = 1.0
+
+bcf.ComputationalGrid.NX = 18
+bcf.ComputationalGrid.NY = 15
+bcf.ComputationalGrid.NZ = 8
+
+# ---------------------------------------------------------
+# The Names of the GeomInputs
+# ---------------------------------------------------------
+bcf.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
+
+# ---------------------------------------------------------
+# Domain Geometry Input
+# ---------------------------------------------------------
+bcf.GeomInput.domain_input.InputType = "Box"
+bcf.GeomInput.domain_input.GeomName = "domain"
+
+# ---------------------------------------------------------
+# Domain Geometry
+# ---------------------------------------------------------
+bcf.Geom.domain.Lower.X = -10.0
+bcf.Geom.domain.Lower.Y = 10.0
+bcf.Geom.domain.Lower.Z = 1.0
+
+bcf.Geom.domain.Upper.X = 150.0
+bcf.Geom.domain.Upper.Y = 170.0
+bcf.Geom.domain.Upper.Z = 9.0
+
+bcf.Geom.domain.Patches = "left right front back bottom top"
+
+# ---------------------------------------------------------
+# Background Geometry Input
+# ---------------------------------------------------------
+bcf.GeomInput.background_input.InputType = "Box"
+bcf.GeomInput.background_input.GeomName = "background"
+
+# ---------------------------------------------------------
+# Background Geometry
+# ---------------------------------------------------------
+bcf.Geom.background.Lower.X = -99999999.0
+bcf.Geom.background.Lower.Y = -99999999.0
+bcf.Geom.background.Lower.Z = -99999999.0
+
+bcf.Geom.background.Upper.X = 99999999.0
+bcf.Geom.background.Upper.Y = 99999999.0
+bcf.Geom.background.Upper.Z = 99999999.0
+
+
+# ---------------------------------------------------------
+# Source_Region Geometry Input
+# ---------------------------------------------------------
+bcf.GeomInput.source_region_input.InputType = "Box"
+bcf.GeomInput.source_region_input.GeomName = "source_region"
+
+# ---------------------------------------------------------
+# Source_Region Geometry
+# ---------------------------------------------------------
+bcf.Geom.source_region.Lower.X = 65.56
+bcf.Geom.source_region.Lower.Y = 79.34
+bcf.Geom.source_region.Lower.Z = 4.5
+
+bcf.Geom.source_region.Upper.X = 74.44
+bcf.Geom.source_region.Upper.Y = 89.99
+bcf.Geom.source_region.Upper.Z = 5.5
+
+
+# ---------------------------------------------------------
+# Concen_Region Geometry Input
+# ---------------------------------------------------------
+bcf.GeomInput.concen_region_input.InputType = "Box"
+bcf.GeomInput.concen_region_input.GeomName = "concen_region"
+
+# ---------------------------------------------------------
+# Concen_Region Geometry
+# ---------------------------------------------------------
+bcf.Geom.concen_region.Lower.X = 60.0
+bcf.Geom.concen_region.Lower.Y = 80.0
+bcf.Geom.concen_region.Lower.Z = 4.0
+
+bcf.Geom.concen_region.Upper.X = 80.0
+bcf.Geom.concen_region.Upper.Y = 100.0
+bcf.Geom.concen_region.Upper.Z = 6.0
+
+# -----------------------------------------------------------------------------
+# Perm
+# -----------------------------------------------------------------------------
+bcf.Geom.Perm.Names = "background"
+
+bcf.Geom.background.Perm.Type = "Constant"
+bcf.Geom.background.Perm.Value = 4.0
+
+bcf.Perm.TensorType = "TensorByGeom"
+
+bcf.Geom.Perm.TensorByGeom.Names = "background"
+
+bcf.Geom.background.Perm.TensorValX = 1.0
+bcf.Geom.background.Perm.TensorValY = 1.0
+bcf.Geom.background.Perm.TensorValZ = 1.0
+
+# -----------------------------------------------------------------------------
+# Specific Storage
+# -----------------------------------------------------------------------------
+
+bcf.SpecificStorage.Type = "Constant"
+bcf.SpecificStorage.GeomNames = "domain"
+bcf.Geom.domain.SpecificStorage.Value = 1.0e-4
+
+# -----------------------------------------------------------------------------
+# Phases
+# -----------------------------------------------------------------------------
+
+bcf.Phase.Names = "water"
+
+bcf.Phase.water.Density.Type = "Constant"
+bcf.Phase.water.Density.Value = 1.0
+
+bcf.Phase.water.Viscosity.Type = "Constant"
+bcf.Phase.water.Viscosity.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Contaminants
+# -----------------------------------------------------------------------------
+bcf.Contaminants.Names = ""
+
+# -----------------------------------------------------------------------------
+# Retardation
+# -----------------------------------------------------------------------------
+bcf.Geom.Retardation.GeomNames = ""
+
+# -----------------------------------------------------------------------------
+# Gravity
+# -----------------------------------------------------------------------------
+
+bcf.Gravity = 1.0
+
+# -----------------------------------------------------------------------------
+# Setup timing info
+# -----------------------------------------------------------------------------
+
+bcf.TimingInfo.BaseUnit = 1.0
+bcf.TimingInfo.StartCount = 0
+bcf.TimingInfo.StartTime = 0.0
+bcf.TimingInfo.StopTime = 0.010
+bcf.TimingInfo.DumpInterval = -1
+bcf.TimeStep.Type = "Constant"
+bcf.TimeStep.Value = 0.001
+
+# -----------------------------------------------------------------------------
+# Porosity
+# -----------------------------------------------------------------------------
+
+bcf.Geom.Porosity.GeomNames = "background"
+
+bcf.Geom.background.Porosity.Type = "Constant"
+bcf.Geom.background.Porosity.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Domain
+# -----------------------------------------------------------------------------
+bcf.Domain.GeomName = "domain"
+
+# -----------------------------------------------------------------------------
+# Relative Permeability
+# -----------------------------------------------------------------------------
+
+bcf.Phase.RelPerm.Type = "VanGenuchten"
+bcf.Phase.RelPerm.GeomNames = "domain"
+bcf.Geom.domain.RelPerm.Alpha = 0.005
+bcf.Geom.domain.RelPerm.N = 2.0
+
+# ---------------------------------------------------------
+# Saturation
+# ---------------------------------------------------------
+
+bcf.Phase.Saturation.Type = "VanGenuchten"
+bcf.Phase.Saturation.GeomNames = "domain"
+bcf.Geom.domain.Saturation.Alpha = 0.005
+bcf.Geom.domain.Saturation.N = 2.0
+bcf.Geom.domain.Saturation.SRes = 0.2
+bcf.Geom.domain.Saturation.SSat = 0.99
+
+# -----------------------------------------------------------------------------
+# Wells
+# -----------------------------------------------------------------------------
+bcf.Wells.Names = ""
+
+# -----------------------------------------------------------------------------
+# Time Cycles
+# -----------------------------------------------------------------------------
+bcf.Cycle.Names = "constant"
+bcf.Cycle.constant.Names = "alltime"
+bcf.Cycle.constant.alltime.Length = 1
+bcf.Cycle.constant.Repeat = -1
+
+# -----------------------------------------------------------------------------
+# Boundary Conditions: Pressure
+# -----------------------------------------------------------------------------
+bcf.BCPressure.PatchNames = "left right front back bottom top"
+
+bcf.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+bcf.Patch.left.BCPressure.Cycle = "constant"
+bcf.Patch.left.BCPressure.RefGeom = "domain"
+bcf.Patch.left.BCPressure.RefPatch = "bottom"
+bcf.Patch.left.BCPressure.alltime.Value = 5.0
+
+bcf.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+bcf.Patch.right.BCPressure.Cycle = "constant"
+bcf.Patch.right.BCPressure.RefGeom = "domain"
+bcf.Patch.right.BCPressure.RefPatch = "bottom"
+bcf.Patch.right.BCPressure.alltime.Value = 3.0
+
+bcf.Patch.front.BCPressure.Type = "FluxConst"
+bcf.Patch.front.BCPressure.Cycle = "constant"
+bcf.Patch.front.BCPressure.alltime.Value = 0.0
+
+bcf.Patch.back.BCPressure.Type = "FluxConst"
+bcf.Patch.back.BCPressure.Cycle = "constant"
+bcf.Patch.back.BCPressure.alltime.Value = 0.0
+
+bcf.Patch.bottom.BCPressure.Type = "FluxConst"
+bcf.Patch.bottom.BCPressure.Cycle = "constant"
+bcf.Patch.bottom.BCPressure.alltime.Value = 0.0
+
+# bcf.Patch.top.BCPressure.Type = 'FluxConst'
+# bcf.Patch.top.BCPressure.Cycle = "constant"
+# bcf.Patch.top.BCPressure.alltime.Value = 0.0
+
+#
+# Testing using the PressureFile option, create a file with 0's for all elements
+flux_filename = "pressure_test.pfb"
+
+flux_array = np.ndarray(
+    (bcf.ComputationalGrid.NX, bcf.ComputationalGrid.NY, bcf.ComputationalGrid.NZ)
+)
+
+write_pfb(
+    new_output_dir_name + "/" + flux_filename,
+    flux_array,
+    bcf.ComputationalGrid.NX,
+    bcf.ComputationalGrid.NY,
+    bcf.ComputationalGrid.NZ,
+    bcf.ComputationalGrid.Lower.X,
+    bcf.ComputationalGrid.Lower.Y,
+    bcf.ComputationalGrid.Lower.Z,
+    bcf.ComputationalGrid.DX,
+    bcf.ComputationalGrid.DY,
+    bcf.ComputationalGrid.DZ,
+    z_first=False,
+)
+
+bcf.Patch.top.BCPressure.Type = "FluxFile"
+bcf.Patch.top.BCPressure.Cycle = "constant"
+bcf.Patch.top.BCPressure.alltime.FileName = flux_filename
+
+# ---------------------------------------------------------
+# Topo slopes in x-direction
+# ---------------------------------------------------------
+
+bcf.TopoSlopesX.Type = "Constant"
+bcf.TopoSlopesX.GeomNames = ""
+
+bcf.TopoSlopesX.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Topo slopes in y-direction
+# ---------------------------------------------------------
+
+bcf.TopoSlopesY.Type = "Constant"
+bcf.TopoSlopesY.GeomNames = ""
+
+bcf.TopoSlopesY.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
+
+bcf.Mannings.Type = "Constant"
+bcf.Mannings.GeomNames = ""
+bcf.Mannings.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Initial conditions: water pressure
+# ---------------------------------------------------------
+
+bcf.ICPressure.Type = "HydroStaticPatch"
+bcf.ICPressure.GeomNames = "domain"
+bcf.Geom.domain.ICPressure.Value = 3.0
+bcf.Geom.domain.ICPressure.RefGeom = "domain"
+bcf.Geom.domain.ICPressure.RefPatch = "bottom"
+
+# -----------------------------------------------------------------------------
+# Phase sources:
+# -----------------------------------------------------------------------------
+
+bcf.PhaseSources.water.Type = "Constant"
+bcf.PhaseSources.water.GeomNames = "background"
+bcf.PhaseSources.water.Geom.background.Value = 0.0
+
+
+# -----------------------------------------------------------------------------
+# Exact solution specification for error calculations
+# -----------------------------------------------------------------------------
+
+bcf.KnownSolution = "NoKnownSolution"
+
+
+# -----------------------------------------------------------------------------
+# Set solver parameters
+# -----------------------------------------------------------------------------
+bcf.Solver = "Richards"
+bcf.Solver.MaxIter = 5
+
+bcf.Solver.Nonlinear.MaxIter = 10
+bcf.Solver.Nonlinear.ResidualTol = 1e-9
+bcf.Solver.Nonlinear.EtaChoice = "EtaConstant"
+bcf.Solver.Nonlinear.EtaValue = 1e-5
+bcf.Solver.Nonlinear.UseJacobian = True
+bcf.Solver.Nonlinear.DerivativeEpsilon = 1e-2
+
+bcf.Solver.Linear.KrylovDimension = 10
+
+bcf.Solver.Linear.Preconditioner = "PFMG"
+# bcf.Solver.Linear.Preconditioner.MGSemi.MaxIter = 1
+# bcf.Solver.Linear.Preconditioner.MGSemi.MaxLevels = 100
+
+bcf.Solver.PrintVelocities = True
+
+# -----------------------------------------------------------------------------
+# Run and Unload the ParFlow output files
+# -----------------------------------------------------------------------------
+bcf.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+sig_digits = 6
+abs_value = 1e-12
+for i in range(0, 6):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.velx.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in x-velocity for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+    filename = f"/{run_name}.out.vely.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in y-velocity for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+    filename = f"/{run_name}.out.vely.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in z-velocity for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+rm(new_output_dir_name)
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/bc_pressure_file.py b/test/python/bc_pressure_file.py
new file mode 100644
index 000000000..3c45fc0e8
--- /dev/null
+++ b/test/python/bc_pressure_file.py
@@ -0,0 +1,426 @@
+# Test for using the "PressureFile" option for BC.
+# Test is based on default_richards with modified BC
+# specification.
+
+import sys
+from parflow import Run
+from parflow.tools.fs import mkdir, get_absolute_path, rm
+from parflow.tools.compare import pf_test_file, pf_test_file_with_abs
+from parflow.tools.io import write_pfb
+import numpy as np
+
+run_name = "bc_pressure_file"
+bcp = Run(run_name, __file__)
+
+new_output_dir_name = get_absolute_path("test_output/bcp")
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+
+bcp.FileVersion = 4
+
+bcp.Process.Topology.P = 1
+bcp.Process.Topology.Q = 1
+bcp.Process.Topology.R = 1
+
+# ---------------------------------------------------------
+# Computational Grid
+# ---------------------------------------------------------
+bcp.ComputationalGrid.Lower.X = -10.0
+bcp.ComputationalGrid.Lower.Y = 10.0
+bcp.ComputationalGrid.Lower.Z = 1.0
+
+bcp.ComputationalGrid.DX = 8.8888888888888893
+bcp.ComputationalGrid.DY = 10.666666666666666
+bcp.ComputationalGrid.DZ = 1.0
+
+bcp.ComputationalGrid.NX = 18
+bcp.ComputationalGrid.NY = 15
+bcp.ComputationalGrid.NZ = 8
+
+# ---------------------------------------------------------
+# The Names of the GeomInputs
+# ---------------------------------------------------------
+bcp.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
+
+# ---------------------------------------------------------
+# Domain Geometry Input
+# ---------------------------------------------------------
+bcp.GeomInput.domain_input.InputType = "Box"
+bcp.GeomInput.domain_input.GeomName = "domain"
+
+# ---------------------------------------------------------
+# Domain Geometry
+# ---------------------------------------------------------
+bcp.Geom.domain.Lower.X = -10.0
+bcp.Geom.domain.Lower.Y = 10.0
+bcp.Geom.domain.Lower.Z = 1.0
+
+bcp.Geom.domain.Upper.X = 150.0
+bcp.Geom.domain.Upper.Y = 170.0
+bcp.Geom.domain.Upper.Z = 9.0
+
+bcp.Geom.domain.Patches = "left right front back bottom top"
+
+# ---------------------------------------------------------
+# Background Geometry Input
+# ---------------------------------------------------------
+bcp.GeomInput.background_input.InputType = "Box"
+bcp.GeomInput.background_input.GeomName = "background"
+
+# ---------------------------------------------------------
+# Background Geometry
+# ---------------------------------------------------------
+bcp.Geom.background.Lower.X = -99999999.0
+bcp.Geom.background.Lower.Y = -99999999.0
+bcp.Geom.background.Lower.Z = -99999999.0
+
+bcp.Geom.background.Upper.X = 99999999.0
+bcp.Geom.background.Upper.Y = 99999999.0
+bcp.Geom.background.Upper.Z = 99999999.0
+
+
+# ---------------------------------------------------------
+# Source_Region Geometry Input
+# ---------------------------------------------------------
+bcp.GeomInput.source_region_input.InputType = "Box"
+bcp.GeomInput.source_region_input.GeomName = "source_region"
+
+# ---------------------------------------------------------
+# Source_Region Geometry
+# ---------------------------------------------------------
+bcp.Geom.source_region.Lower.X = 65.56
+bcp.Geom.source_region.Lower.Y = 79.34
+bcp.Geom.source_region.Lower.Z = 4.5
+
+bcp.Geom.source_region.Upper.X = 74.44
+bcp.Geom.source_region.Upper.Y = 89.99
+bcp.Geom.source_region.Upper.Z = 5.5
+
+
+# ---------------------------------------------------------
+# Concen_Region Geometry Input
+# ---------------------------------------------------------
+bcp.GeomInput.concen_region_input.InputType = "Box"
+bcp.GeomInput.concen_region_input.GeomName = "concen_region"
+
+# ---------------------------------------------------------
+# Concen_Region Geometry
+# ---------------------------------------------------------
+bcp.Geom.concen_region.Lower.X = 60.0
+bcp.Geom.concen_region.Lower.Y = 80.0
+bcp.Geom.concen_region.Lower.Z = 4.0
+
+bcp.Geom.concen_region.Upper.X = 80.0
+bcp.Geom.concen_region.Upper.Y = 100.0
+bcp.Geom.concen_region.Upper.Z = 6.0
+
+# -----------------------------------------------------------------------------
+# Perm
+# -----------------------------------------------------------------------------
+bcp.Geom.Perm.Names = "background"
+
+bcp.Geom.background.Perm.Type = "Constant"
+bcp.Geom.background.Perm.Value = 4.0
+
+bcp.Perm.TensorType = "TensorByGeom"
+
+bcp.Geom.Perm.TensorByGeom.Names = "background"
+
+bcp.Geom.background.Perm.TensorValX = 1.0
+bcp.Geom.background.Perm.TensorValY = 1.0
+bcp.Geom.background.Perm.TensorValZ = 1.0
+
+# -----------------------------------------------------------------------------
+# Specific Storage
+# -----------------------------------------------------------------------------
+
+bcp.SpecificStorage.Type = "Constant"
+bcp.SpecificStorage.GeomNames = "domain"
+bcp.Geom.domain.SpecificStorage.Value = 1.0e-4
+
+# -----------------------------------------------------------------------------
+# Phases
+# -----------------------------------------------------------------------------
+
+bcp.Phase.Names = "water"
+
+bcp.Phase.water.Density.Type = "Constant"
+bcp.Phase.water.Density.Value = 1.0
+
+bcp.Phase.water.Viscosity.Type = "Constant"
+bcp.Phase.water.Viscosity.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Contaminants
+# -----------------------------------------------------------------------------
+bcp.Contaminants.Names = ""
+
+# -----------------------------------------------------------------------------
+# Retardation
+# -----------------------------------------------------------------------------
+bcp.Geom.Retardation.GeomNames = ""
+
+# -----------------------------------------------------------------------------
+# Gravity
+# -----------------------------------------------------------------------------
+
+bcp.Gravity = 1.0
+
+# -----------------------------------------------------------------------------
+# Setup timing info
+# -----------------------------------------------------------------------------
+
+bcp.TimingInfo.BaseUnit = 1.0
+bcp.TimingInfo.StartCount = 0
+bcp.TimingInfo.StartTime = 0.0
+bcp.TimingInfo.StopTime = 0.010
+bcp.TimingInfo.DumpInterval = -1
+bcp.TimeStep.Type = "Constant"
+bcp.TimeStep.Value = 0.001
+
+# -----------------------------------------------------------------------------
+# Porosity
+# -----------------------------------------------------------------------------
+
+bcp.Geom.Porosity.GeomNames = "background"
+
+bcp.Geom.background.Porosity.Type = "Constant"
+bcp.Geom.background.Porosity.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Domain
+# -----------------------------------------------------------------------------
+bcp.Domain.GeomName = "domain"
+
+# -----------------------------------------------------------------------------
+# Relative Permeability
+# -----------------------------------------------------------------------------
+
+bcp.Phase.RelPerm.Type = "VanGenuchten"
+bcp.Phase.RelPerm.GeomNames = "domain"
+bcp.Geom.domain.RelPerm.Alpha = 0.005
+bcp.Geom.domain.RelPerm.N = 2.0
+
+# ---------------------------------------------------------
+# Saturation
+# ---------------------------------------------------------
+
+bcp.Phase.Saturation.Type = "VanGenuchten"
+bcp.Phase.Saturation.GeomNames = "domain"
+bcp.Geom.domain.Saturation.Alpha = 0.005
+bcp.Geom.domain.Saturation.N = 2.0
+bcp.Geom.domain.Saturation.SRes = 0.2
+bcp.Geom.domain.Saturation.SSat = 0.99
+
+# -----------------------------------------------------------------------------
+# Wells
+# -----------------------------------------------------------------------------
+bcp.Wells.Names = ""
+
+# -----------------------------------------------------------------------------
+# Time Cycles
+# -----------------------------------------------------------------------------
+bcp.Cycle.Names = "constant"
+bcp.Cycle.constant.Names = "alltime"
+bcp.Cycle.constant.alltime.Length = 1
+bcp.Cycle.constant.Repeat = -1
+
+# -----------------------------------------------------------------------------
+# Boundary Conditions: Pressure
+# -----------------------------------------------------------------------------
+bcp.BCPressure.PatchNames = "left right front back bottom top"
+
+bcp.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+bcp.Patch.left.BCPressure.Cycle = "constant"
+bcp.Patch.left.BCPressure.RefGeom = "domain"
+bcp.Patch.left.BCPressure.RefPatch = "bottom"
+bcp.Patch.left.BCPressure.alltime.Value = 5.0
+
+bcp.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+bcp.Patch.right.BCPressure.Cycle = "constant"
+bcp.Patch.right.BCPressure.RefGeom = "domain"
+bcp.Patch.right.BCPressure.RefPatch = "bottom"
+bcp.Patch.right.BCPressure.alltime.Value = 3.0
+
+bcp.Patch.front.BCPressure.Type = "FluxConst"
+bcp.Patch.front.BCPressure.Cycle = "constant"
+bcp.Patch.front.BCPressure.alltime.Value = 0.0
+
+bcp.Patch.back.BCPressure.Type = "FluxConst"
+bcp.Patch.back.BCPressure.Cycle = "constant"
+bcp.Patch.back.BCPressure.alltime.Value = 0.0
+
+bcp.Patch.bottom.BCPressure.Type = "FluxConst"
+bcp.Patch.bottom.BCPressure.Cycle = "constant"
+bcp.Patch.bottom.BCPressure.alltime.Value = 0.0
+
+# Testing using the PressureFile option, create a file with 0's for all elements
+pressure_filename = "pressure_test.pfb"
+
+pressure_array = np.ndarray(
+    (bcp.ComputationalGrid.NX, bcp.ComputationalGrid.NY, bcp.ComputationalGrid.NZ)
+)
+
+write_pfb(
+    new_output_dir_name + "/" + pressure_filename,
+    pressure_array,
+    bcp.ComputationalGrid.NX,
+    bcp.ComputationalGrid.NY,
+    bcp.ComputationalGrid.NZ,
+    bcp.ComputationalGrid.Lower.X,
+    bcp.ComputationalGrid.Lower.Y,
+    bcp.ComputationalGrid.Lower.Z,
+    bcp.ComputationalGrid.DX,
+    bcp.ComputationalGrid.DY,
+    bcp.ComputationalGrid.DZ,
+    z_first=False,
+)
+
+bcp.Patch.top.BCPressure.Type = "PressureFile"
+bcp.Patch.top.BCPressure.Cycle = "constant"
+bcp.Patch.top.BCPressure.alltime.FileName = pressure_filename
+
+# ---------------------------------------------------------
+# Topo slopes in x-direction
+# ---------------------------------------------------------
+
+bcp.TopoSlopesX.Type = "Constant"
+bcp.TopoSlopesX.GeomNames = ""
+
+bcp.TopoSlopesX.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Topo slopes in y-direction
+# ---------------------------------------------------------
+
+bcp.TopoSlopesY.Type = "Constant"
+bcp.TopoSlopesY.GeomNames = ""
+
+bcp.TopoSlopesY.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
+
+bcp.Mannings.Type = "Constant"
+bcp.Mannings.GeomNames = ""
+bcp.Mannings.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Initial conditions: water pressure
+# ---------------------------------------------------------
+
+bcp.ICPressure.Type = "HydroStaticPatch"
+bcp.ICPressure.GeomNames = "domain"
+bcp.Geom.domain.ICPressure.Value = 3.0
+bcp.Geom.domain.ICPressure.RefGeom = "domain"
+bcp.Geom.domain.ICPressure.RefPatch = "bottom"
+
+# -----------------------------------------------------------------------------
+# Phase sources:
+# -----------------------------------------------------------------------------
+
+bcp.PhaseSources.water.Type = "Constant"
+bcp.PhaseSources.water.GeomNames = "background"
+bcp.PhaseSources.water.Geom.background.Value = 0.0
+
+
+# -----------------------------------------------------------------------------
+# Exact solution specification for error calculations
+# -----------------------------------------------------------------------------
+
+bcp.KnownSolution = "NoKnownSolution"
+
+
+# -----------------------------------------------------------------------------
+# Set solver parameters
+# -----------------------------------------------------------------------------
+bcp.Solver = "Richards"
+bcp.Solver.MaxIter = 5
+
+bcp.Solver.Nonlinear.MaxIter = 10
+bcp.Solver.Nonlinear.ResidualTol = 1e-9
+bcp.Solver.Nonlinear.EtaChoice = "EtaConstant"
+bcp.Solver.Nonlinear.EtaValue = 1e-5
+bcp.Solver.Nonlinear.UseJacobian = True
+bcp.Solver.Nonlinear.DerivativeEpsilon = 1e-2
+
+bcp.Solver.Linear.KrylovDimension = 10
+
+bcp.Solver.Linear.Preconditioner = "PFMG"
+# bcp.Solver.Linear.Preconditioner.MGSemi.MaxIter = 1
+# bcp.Solver.Linear.Preconditioner.MGSemi.MaxLevels = 100
+
+bcp.Solver.PrintVelocities = True
+
+# -----------------------------------------------------------------------------
+# Run and Unload the ParFlow output files
+# -----------------------------------------------------------------------------
+bcp.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+sig_digits = 6
+abs_value = 1e-12
+for i in range(0, 6):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.velx.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in x-velocity for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+    filename = f"/{run_name}.out.vely.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in y-velocity for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+    filename = f"/{run_name}.out.vely.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in z-velocity for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+rm(new_output_dir_name)
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/channelwidth.py b/test/python/channelwidth.py
new file mode 100644
index 000000000..f7dc2d875
--- /dev/null
+++ b/test/python/channelwidth.py
@@ -0,0 +1,409 @@
+# Import the ParFlow package
+import os
+import sys
+from parflow import Run, write_pfb
+import shutil
+from parflow.tools.fs import mkdir, cp, chdir, get_absolute_path, rm, exists
+from parflow.tools.io import read_pfb
+import numpy as np
+
+# Set our Run Name
+run_name = "channel_width_example"
+wc_test = Run(run_name, __file__)
+
+# -----------------------------------------------------------------------------
+# File input version number
+# -----------------------------------------------------------------------------
+wc_test.FileVersion = 4
+
+# -----------------------------------------------------------------------------
+# Process Topology
+# -----------------------------------------------------------------------------
+wc_test.Process.Topology.P = 1
+wc_test.Process.Topology.Q = 1
+wc_test.Process.Topology.R = 1
+
+# -----------------------------------------------------------------------------
+# Computational Grid
+# -----------------------------------------------------------------------------
+wc_test.ComputationalGrid.Lower.X = 0.0
+wc_test.ComputationalGrid.Lower.Y = 0.0
+wc_test.ComputationalGrid.Lower.Z = 0.0
+
+wc_test.ComputationalGrid.DX = 100.0
+wc_test.ComputationalGrid.DY = 2.0
+wc_test.ComputationalGrid.DZ = 1.0
+
+wc_test.ComputationalGrid.NX = 20
+wc_test.ComputationalGrid.NY = 1
+wc_test.ComputationalGrid.NZ = 10
+
+# -----------------------------------------------------------------------------
+# The Names of the GeomInputs
+# -----------------------------------------------------------------------------
+wc_test.GeomInput.Names = "domain_input"
+
+# -----------------------------------------------------------------------------
+# Domain Geometry Input
+# -----------------------------------------------------------------------------
+wc_test.GeomInput.domain_input.InputType = "Box"
+wc_test.GeomInput.domain_input.GeomName = "domain"
+
+# -----------------------------------------------------------------------------
+# Domain Geometry
+# -----------------------------------------------------------------------------
+wc_test.Geom.domain.Lower.X = 0.0
+wc_test.Geom.domain.Lower.Y = 0.0
+wc_test.Geom.domain.Lower.Z = 0.0
+
+wc_test.Geom.domain.Upper.X = 2000.0
+wc_test.Geom.domain.Upper.Y = 2.0
+wc_test.Geom.domain.Upper.Z = 10.0
+
+wc_test.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
+
+
+# --------------------------------------------
+# Variable dz Assignments
+# ------------------------------------------
+wc_test.Solver.Nonlinear.VariableDz = False
+wc_test.dzScale.GeomNames = "domain"
+wc_test.dzScale.Type = "nzList"
+wc_test.dzScale.nzListNumber = 10
+
+# cells start at the bottom (0) and moves up to the top
+# domain is 49 m thick, root zone is down to 4 cells
+# so the root zone is 2 m thick
+wc_test.Cell._0.dzScale.Value = 10.0  # 10* 1.0 = 10  m  layer
+wc_test.Cell._1.dzScale.Value = 10.0
+wc_test.Cell._2.dzScale.Value = 10.0
+wc_test.Cell._3.dzScale.Value = 10.0
+wc_test.Cell._4.dzScale.Value = 5.0
+wc_test.Cell._5.dzScale.Value = 1.0
+wc_test.Cell._6.dzScale.Value = 1.0
+wc_test.Cell._7.dzScale.Value = 0.6  # 0.6* 1.0 = 0.6  60 cm 3rd layer
+wc_test.Cell._8.dzScale.Value = 0.3  # 0.3* 1.0 = 0.3  30 cm 2nd layer
+wc_test.Cell._9.dzScale.Value = 0.1  # 0.1* 1.0 = 0.1  10 cm top layer
+
+# -----------------------------------------------------------------------------
+# Perm
+# -----------------------------------------------------------------------------
+wc_test.Geom.Perm.Names = "domain"
+wc_test.Geom.domain.Perm.Type = "Constant"
+wc_test.Geom.domain.Perm.Value = 0.01465  # m/h
+
+wc_test.Perm.TensorType = "TensorByGeom"
+wc_test.Geom.Perm.TensorByGeom.Names = "domain"
+wc_test.Geom.domain.Perm.TensorValX = 1.0
+wc_test.Geom.domain.Perm.TensorValY = 1.0
+wc_test.Geom.domain.Perm.TensorValZ = 1.0
+
+# -----------------------------------------------------------------------------
+# Specific Storage
+# -----------------------------------------------------------------------------
+wc_test.SpecificStorage.Type = "Constant"
+wc_test.SpecificStorage.GeomNames = "domain"
+wc_test.Geom.domain.SpecificStorage.Value = 1.0e-4
+
+# -----------------------------------------------------------------------------
+# Phases
+# -----------------------------------------------------------------------------
+wc_test.Phase.Names = "water"
+
+wc_test.Phase.water.Density.Type = "Constant"
+wc_test.Phase.water.Density.Value = 1.0
+
+wc_test.Phase.water.Viscosity.Type = "Constant"
+wc_test.Phase.water.Viscosity.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Contaminants
+# -----------------------------------------------------------------------------
+wc_test.Contaminants.Names = ""
+
+
+# -----------------------------------------------------------------------------
+# Gravity
+# -----------------------------------------------------------------------------
+wc_test.Gravity = 1.0
+
+# -----------------------------------------------------------------------------
+# Setup Timing
+# -----------------------------------------------------------------------------
+wc_test.TimingInfo.BaseUnit = 1.0
+wc_test.TimingInfo.StartCount = 0
+wc_test.TimingInfo.StartTime = 0.0
+wc_test.TimingInfo.StopTime = 5.0
+wc_test.TimingInfo.DumpInterval = 1.0
+wc_test.TimeStep.Type = "Constant"
+wc_test.TimeStep.Value = 1.0
+
+
+# -----------------------------------------------------------------------------
+# Porosity
+# -----------------------------------------------------------------------------
+wc_test.Geom.Porosity.GeomNames = "domain"
+
+wc_test.Geom.domain.Porosity.Type = "Constant"
+wc_test.Geom.domain.Porosity.Value = 0.25
+
+# -----------------------------------------------------------------------------
+# Domain
+# -----------------------------------------------------------------------------
+wc_test.Domain.GeomName = "domain"
+
+# -----------------------------------------------------------------------------
+# Mobility
+# -----------------------------------------------------------------------------
+wc_test.Phase.water.Mobility.Type = "Constant"
+wc_test.Phase.water.Mobility.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Relative Permeability
+# -----------------------------------------------------------------------------
+wc_test.Phase.RelPerm.Type = "VanGenuchten"
+wc_test.Phase.RelPerm.GeomNames = "domain"
+
+wc_test.Geom.domain.RelPerm.Alpha = 1.0
+wc_test.Geom.domain.RelPerm.N = 2.0
+
+# ---------------------------------------------------------
+# Saturation
+# ---------------------------------------------------------
+wc_test.Phase.Saturation.Type = "VanGenuchten"
+wc_test.Phase.Saturation.GeomNames = "domain"
+
+wc_test.Geom.domain.Saturation.Alpha = 1.0
+wc_test.Geom.domain.Saturation.N = 2.0
+wc_test.Geom.domain.Saturation.SRes = 0.2
+wc_test.Geom.domain.Saturation.SSat = 1.0
+
+# -----------------------------------------------------------------------------
+# Wells
+# -----------------------------------------------------------------------------
+wc_test.Wells.Names = ""
+
+
+# -----------------------------------------------------------------------------
+# Time Cycles
+# -----------------------------------------------------------------------------
+wc_test.Cycle.Names = "constant"
+wc_test.Cycle.constant.Names = "alltime"
+wc_test.Cycle.constant.alltime.Length = 1
+wc_test.Cycle.constant.Repeat = -1
+
+# -----------------------------------------------------------------------------
+# Boundary Conditions: Pressure
+# -----------------------------------------------------------------------------
+wc_test.BCPressure.PatchNames = "x_lower x_upper y_lower y_upper z_lower z_upper"
+
+wc_test.Patch.y_lower.BCPressure.Type = "FluxConst"
+wc_test.Patch.y_lower.BCPressure.Cycle = "constant"
+wc_test.Patch.y_lower.BCPressure.alltime.Value = 0.0
+
+wc_test.Patch.z_lower.BCPressure.Type = "FluxConst"
+wc_test.Patch.z_lower.BCPressure.Cycle = "constant"
+wc_test.Patch.z_lower.BCPressure.alltime.Value = 0.0
+
+wc_test.Patch.x_lower.BCPressure.Type = "FluxConst"
+wc_test.Patch.x_lower.BCPressure.Cycle = "constant"
+wc_test.Patch.x_lower.BCPressure.alltime.Value = 0.0
+
+wc_test.Patch.x_upper.BCPressure.Type = "DirEquilRefPatch"
+wc_test.Patch.x_upper.BCPressure.RefGeom = "domain"
+wc_test.Patch.x_upper.BCPressure.RefPatch = "z_upper"
+wc_test.Patch.x_upper.BCPressure.Cycle = "constant"
+wc_test.Patch.x_upper.BCPressure.alltime.Value = (
+    -1.0
+)  # ocean boundary is 1m below land surface
+
+wc_test.Patch.y_upper.BCPressure.Type = "FluxConst"
+wc_test.Patch.y_upper.BCPressure.Cycle = "constant"
+wc_test.Patch.y_upper.BCPressure.alltime.Value = 0.0
+
+wc_test.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+wc_test.Patch.z_upper.BCPressure.Cycle = "constant"
+wc_test.Patch.z_upper.BCPressure.alltime.Value = -0.01
+
+# ---------------------------------------------------------
+# Topo slopes in x-direction
+# ---------------------------------------------------------
+wc_test.TopoSlopesX.Type = "Constant"
+wc_test.TopoSlopesX.GeomNames = "domain"
+wc_test.TopoSlopesX.Geom.domain.Value = (
+    -0.1
+)  # slope in X-direction to allow ponded water to run off
+
+# ---------------------------------------------------------
+# Topo slopes in y-direction
+# ---------------------------------------------------------
+wc_test.TopoSlopesY.Type = "Constant"
+wc_test.TopoSlopesY.GeomNames = "domain"
+wc_test.TopoSlopesY.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
+wc_test.Mannings.Type = "Constant"
+wc_test.Mannings.GeomNames = "domain"
+wc_test.Mannings.Geom.domain.Value = 2.0e-6
+
+# -----------------------------------------------------------------------------
+# Phase sources:
+# -----------------------------------------------------------------------------
+wc_test.PhaseSources.water.Type = "Constant"
+wc_test.PhaseSources.water.GeomNames = "domain"
+wc_test.PhaseSources.water.Geom.domain.Value = 0.0
+
+# -----------------------------------------------------------------------------
+# Exact solution specification for error calculations
+# -----------------------------------------------------------------------------
+wc_test.KnownSolution = "NoKnownSolution"
+
+# -----------------------------------------------------------------------------
+# Set solver parameters
+# -----------------------------------------------------------------------------
+wc_test.Solver = "Richards"
+wc_test.Solver.MaxIter = 9000
+
+wc_test.Solver.Nonlinear.MaxIter = 100
+wc_test.Solver.Nonlinear.ResidualTol = 1e-5
+wc_test.Solver.Nonlinear.EtaChoice = "Walker1"
+wc_test.Solver.Nonlinear.EtaValue = 0.01
+wc_test.Solver.Nonlinear.UseJacobian = True
+wc_test.Solver.Nonlinear.DerivativeEpsilon = 1e-12
+wc_test.Solver.Nonlinear.StepTol = 1e-30
+wc_test.Solver.Nonlinear.Globalization = "LineSearch"
+wc_test.Solver.Linear.KrylovDimension = 100
+wc_test.Solver.Linear.MaxRestarts = 5
+wc_test.Solver.Linear.Preconditioner = "PFMG"
+wc_test.Solver.PrintSubsurf = True
+wc_test.Solver.Drop = 1e-20
+wc_test.Solver.AbsTol = 1e-9
+
+# Writing output options for ParFlow
+should_write = True  # only PFB output for water balance example
+#  PFB  no SILO
+wc_test.Solver.PrintSubsurfData = should_write
+wc_test.Solver.PrintPressure = should_write
+wc_test.Solver.PrintSaturation = should_write
+wc_test.Solver.PrintCLM = should_write
+wc_test.Solver.PrintMask = should_write
+wc_test.Solver.PrintSpecificStorage = should_write
+wc_test.Solver.PrintEvapTrans = should_write
+wc_test.Solver.PrintVelocities = True
+wc_test.Solver.PrintChannelWidth = True
+
+
+# ---------------------------------------------------
+# LSM / CLM options
+# ---------------------------------------------------
+
+# Writing output options for CLM
+# no native CLM logs
+wc_test.Solver.PrintLSMSink = False
+wc_test.Solver.CLM.CLMDumpInterval = 1
+wc_test.Solver.CLM.CLMFileDir = "output/"
+wc_test.Solver.CLM.BinaryOutDir = False
+wc_test.Solver.CLM.IstepStart = 1
+wc_test.Solver.WriteCLMBinary = False
+wc_test.Solver.CLM.WriteLogs = False
+wc_test.Solver.CLM.WriteLastRST = True
+wc_test.Solver.CLM.DailyRST = False
+wc_test.Solver.CLM.SingleFile = True
+
+
+# ---------------------------------------------------
+# Initial conditions: water pressure
+# ---------------------------------------------------
+wc_test.ICPressure.Type = "HydroStaticPatch"
+wc_test.ICPressure.GeomNames = "domain"
+wc_test.Geom.domain.ICPressure.Value = -10.00
+wc_test.Geom.domain.ICPressure.RefGeom = "domain"
+wc_test.Geom.domain.ICPressure.RefPatch = "z_upper"
+
+# ---------------------------------------------------------
+# Test channel width with constant input
+# ---------------------------------------------------------
+
+# ---------------------------------------------------------
+# Channel width in x-direction
+# ---------------------------------------------------------
+wc_test.Solver.Nonlinear.ChannelWidthExistX = True
+wc_test.ChannelWidthX.Type = "Constant"
+wc_test.ChannelWidthX.GeomNames = "domain"
+wc_test.ChannelWidthX.Geom.domain.Value = 1.0
+
+# ---------------------------------------------------------
+# Channel width in y-direction
+# ---------------------------------------------------------
+wc_test.Solver.Nonlinear.ChannelWidthExistY = True
+wc_test.ChannelWidthY.Type = "Constant"
+wc_test.ChannelWidthY.GeomNames = "domain"
+wc_test.ChannelWidthY.Geom.domain.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Run ParFlow
+# -----------------------------------------------------------------------------
+base = os.path.join(os.getcwd(), "output/constant")
+mkdir(base)
+print(f"base: {base}")
+wc_test.run(working_directory=base)
+
+# -----------------------------------------------------------------------------
+# Check if values are as expected
+# -----------------------------------------------------------------------------
+
+data_wcx = read_pfb(os.path.join(base, "channel_width_example.out.wc_x.pfb"))
+data_wcy = read_pfb(os.path.join(base, "channel_width_example.out.wc_y.pfb"))
+
+ones_array = np.ones((1, 1, 20))
+
+if not np.array_equal(data_wcx, ones_array) or not np.array_equal(data_wcy, ones_array):
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
+
+# ---------------------------------------------------------
+# Test channel width with PFB input
+# ---------------------------------------------------------
+
+base = os.path.join(os.getcwd(), "output/pfb")
+mkdir(base)
+print(f"base: {base}")
+write_pfb(get_absolute_path(base + "/Channel_Width_X.pfb"), ones_array)
+write_pfb(get_absolute_path(base + "/Channel_Width_Y.pfb"), ones_array)
+
+# ---------------------------------------------------------
+# Channel width in x-direction
+# ---------------------------------------------------------
+wc_test.Solver.Nonlinear.ChannelWidthExistX = True
+wc_test.ChannelWidthX.Type = "PFBFile"
+wc_test.ChannelWidthX.GeomNames = "domain"
+wc_test.ChannelWidthX.FileName = "Channel_Width_X.pfb"
+
+# ---------------------------------------------------------
+# Channel width in y-direction
+# ---------------------------------------------------------
+wc_test.Solver.Nonlinear.ChannelWidthExistY = True
+wc_test.ChannelWidthY.Type = "PFBFile"
+wc_test.ChannelWidthY.GeomNames = "domain"
+wc_test.ChannelWidthY.FileName = "Channel_Width_Y.pfb"
+
+# -----------------------------------------------------------------------------
+# Run ParFlow
+# -----------------------------------------------------------------------------
+
+wc_test.run(working_directory=base)
+
+# -----------------------------------------------------------------------------
+# Check if values are as expected
+# -----------------------------------------------------------------------------
+
+data_wcx = read_pfb(os.path.join(base, "channel_width_example.out.wc_x.pfb"))
+data_wcy = read_pfb(os.path.join(base, "channel_width_example.out.wc_y.pfb"))
+
+if not np.array_equal(data_wcx, ones_array) or not np.array_equal(data_wcy, ones_array):
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
+
+# Do same checks as in constant case (read wc_y & y and check they are right)
diff --git a/test/python/clm/clm-reuse.py b/test/python/clm/clm-reuse.py
index 6903249c3..4631aa856 100644
--- a/test/python/clm/clm-reuse.py
+++ b/test/python/clm/clm-reuse.py
@@ -1,52 +1,61 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Test CLM with multiple reuse values for input.  The same test is run
 # with different timesteps and reuse values set to match.   E.G. 1s = reuse 1, 0.1s = reuse 10.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
+import sys
+import os
+import math
 from parflow import Run
-from parflow.tools.fs import cp, mkdir, get_absolute_path
+from parflow.tools.fs import cp, mkdir, get_absolute_path, rm
+from parflow.tools.io import read_pfb
 
-clm = Run("clm_reuse", __file__)
+run_name = "reuse"
 
-#-----------------------------------------------------------------------------
+clm = Run(run_name, __file__)
+
+# -----------------------------------------------------------------------------
 # Copying input files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/clm_reuse')
+dir_name = get_absolute_path("test_output/clm_reuse")
 mkdir(dir_name)
 
-cp('$PF_SRC/test/tcl/clm/clm-reuse/drv_clmin.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/clm-reuse/drv_vegm.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/clm-reuse/drv_vegp.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/clm-reuse/forcing_1.txt', dir_name)
+cp("$PF_SRC/test/tcl/clm/clm-reuse/drv_clmin.dat", dir_name)
+cp("$PF_SRC/test/tcl/clm/clm-reuse/drv_vegm.dat", dir_name)
+cp("$PF_SRC/test/tcl/clm/clm-reuse/drv_vegp.dat", dir_name)
+cp("$PF_SRC/test/tcl/clm/clm-reuse/forcing_1.txt", dir_name)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setting test variables
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 # Total runtime of simulation
 stopt = 100
 
 # Reuse values to run with
-reuseValues = [1,4]
+reuseValues = [1, 4]
+
+# This was set for reuse = 4 test; other reuse values will fail
+relativeErrorTolerance = 0.2
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # File input version number
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Process Topology
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.Process.Topology.P = 1
 clm.Process.Topology.Q = 1
 clm.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.ComputationalGrid.Lower.X = 0.0
 clm.ComputationalGrid.Lower.Y = 0.0
@@ -67,253 +76,253 @@
 nz = clm.ComputationalGrid.NZ
 dz = clm.ComputationalGrid.DZ
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # The Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.GeomInput.Names = 'domain_input'
+clm.GeomInput.Names = "domain_input"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.GeomInput.domain_input.InputType = 'Box'
-clm.GeomInput.domain_input.GeomName = 'domain'
+clm.GeomInput.domain_input.InputType = "Box"
+clm.GeomInput.domain_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.Geom.domain.Lower.X = 0.0
 clm.Geom.domain.Lower.Y = 0.0
 clm.Geom.domain.Lower.Z = 0.0
 
-clm.Geom.domain.Upper.X = (nx * dx)
-clm.Geom.domain.Upper.Y = (ny * dy)
-clm.Geom.domain.Upper.Z = (nz * dz)
+clm.Geom.domain.Upper.X = nx * dx
+clm.Geom.domain.Upper.Y = ny * dy
+clm.Geom.domain.Upper.Z = nz * dz
 
-clm.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+clm.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Geom.Perm.Names = 'domain'
+clm.Geom.Perm.Names = "domain"
 
-clm.Geom.domain.Perm.Type = 'Constant'
+clm.Geom.domain.Perm.Type = "Constant"
 clm.Geom.domain.Perm.Value = 0.04465
 
-clm.Perm.TensorType = 'TensorByGeom'
+clm.Perm.TensorType = "TensorByGeom"
 
-clm.Geom.Perm.TensorByGeom.Names = 'domain'
+clm.Geom.Perm.TensorByGeom.Names = "domain"
 
 clm.Geom.domain.Perm.TensorValX = 1.0
 clm.Geom.domain.Perm.TensorValY = 1.0
 clm.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # specific storage does not figure into the impes (fully sat) case but we still
 # need a key for it
 
-clm.SpecificStorage.Type = 'Constant'
-clm.SpecificStorage.GeomNames = 'domain'
+clm.SpecificStorage.Type = "Constant"
+clm.SpecificStorage.GeomNames = "domain"
 clm.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Phase.Names = 'water'
+clm.Phase.Names = "water"
 
-clm.Phase.water.Density.Type = 'Constant'
+clm.Phase.water.Density.Type = "Constant"
 clm.Phase.water.Density.Value = 1.0
 
-clm.Phase.water.Viscosity.Type = 'Constant'
+clm.Phase.water.Viscosity.Type = "Constant"
 clm.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Contaminants.Names = ''
+clm.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.TimingInfo.BaseUnit = 1.0
 clm.TimingInfo.StartCount = 0
 clm.TimingInfo.StartTime = 0.0
 clm.TimingInfo.StopTime = stopt
 clm.TimingInfo.DumpInterval = 1.0
-clm.TimeStep.Type = 'Constant'
+clm.TimeStep.Type = "Constant"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Geom.Porosity.GeomNames = 'domain'
-clm.Geom.domain.Porosity.Type = 'Constant'
+clm.Geom.Porosity.GeomNames = "domain"
+clm.Geom.domain.Porosity.Type = "Constant"
 clm.Geom.domain.Porosity.Value = 0.5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Domain.GeomName = 'domain'
+clm.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mobility
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Phase.water.Mobility.Type = 'Constant'
+clm.Phase.water.Mobility.Type = "Constant"
 clm.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Phase.RelPerm.Type = 'VanGenuchten'
-clm.Phase.RelPerm.GeomNames = 'domain'
+clm.Phase.RelPerm.Type = "VanGenuchten"
+clm.Phase.RelPerm.GeomNames = "domain"
 
 clm.Geom.domain.RelPerm.Alpha = 2.0
 clm.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.Phase.Saturation.Type = 'VanGenuchten'
-clm.Phase.Saturation.GeomNames = 'domain'
+clm.Phase.Saturation.Type = "VanGenuchten"
+clm.Phase.Saturation.GeomNames = "domain"
 
 clm.Geom.domain.Saturation.Alpha = 2.0
 clm.Geom.domain.Saturation.N = 3.0
 clm.Geom.domain.Saturation.SRes = 0.2
 clm.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Wells.Names = ''
+clm.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Cycle.Names = 'constant'
-clm.Cycle.constant.Names = 'alltime'
+clm.Cycle.Names = "constant"
+clm.Cycle.constant.Names = "alltime"
 clm.Cycle.constant.alltime.Length = 1
 clm.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.BCPressure.PatchNames = clm.Geom.domain.Patches
 
-clm.Patch.x_lower.BCPressure.Type = 'FluxConst'
-clm.Patch.x_lower.BCPressure.Cycle = 'constant'
+clm.Patch.x_lower.BCPressure.Type = "FluxConst"
+clm.Patch.x_lower.BCPressure.Cycle = "constant"
 clm.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-clm.Patch.y_lower.BCPressure.Type = 'FluxConst'
-clm.Patch.y_lower.BCPressure.Cycle = 'constant'
+clm.Patch.y_lower.BCPressure.Type = "FluxConst"
+clm.Patch.y_lower.BCPressure.Cycle = "constant"
 clm.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-clm.Patch.z_lower.BCPressure.Type = 'FluxConst'
-clm.Patch.z_lower.BCPressure.Cycle = 'constant'
+clm.Patch.z_lower.BCPressure.Type = "FluxConst"
+clm.Patch.z_lower.BCPressure.Cycle = "constant"
 clm.Patch.z_lower.BCPressure.alltime.Value = -0.00
 
-clm.Patch.x_upper.BCPressure.Type = 'FluxConst'
-clm.Patch.x_upper.BCPressure.Cycle = 'constant'
+clm.Patch.x_upper.BCPressure.Type = "FluxConst"
+clm.Patch.x_upper.BCPressure.Cycle = "constant"
 clm.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-clm.Patch.y_upper.BCPressure.Type = 'FluxConst'
-clm.Patch.y_upper.BCPressure.Cycle = 'constant'
+clm.Patch.y_upper.BCPressure.Type = "FluxConst"
+clm.Patch.y_upper.BCPressure.Cycle = "constant"
 clm.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-clm.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-clm.Patch.z_upper.BCPressure.Cycle = 'constant'
+clm.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+clm.Patch.z_upper.BCPressure.Cycle = "constant"
 clm.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.TopoSlopesX.Type = 'Constant'
-clm.TopoSlopesX.GeomNames = 'domain'
+clm.TopoSlopesX.Type = "Constant"
+clm.TopoSlopesX.GeomNames = "domain"
 clm.TopoSlopesX.Geom.domain.Value = 0.005
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.TopoSlopesY.Type = 'Constant'
-clm.TopoSlopesY.GeomNames = 'domain'
+clm.TopoSlopesY.Type = "Constant"
+clm.TopoSlopesY.GeomNames = "domain"
 clm.TopoSlopesY.Geom.domain.Value = 0.00
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.Mannings.Type = 'Constant'
-clm.Mannings.GeomNames = 'domain'
+clm.Mannings.Type = "Constant"
+clm.Mannings.GeomNames = "domain"
 clm.Mannings.Geom.domain.Value = 1e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.PhaseSources.water.Type = 'Constant'
-clm.PhaseSources.water.GeomNames = 'domain'
+clm.PhaseSources.water.Type = "Constant"
+clm.PhaseSources.water.GeomNames = "domain"
 clm.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.KnownSolution = 'NoKnownSolution'
+clm.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Solver = 'Richards'
+clm.Solver = "Richards"
 clm.Solver.MaxIter = 90000
 
 clm.Solver.Nonlinear.MaxIter = 100
 clm.Solver.Nonlinear.ResidualTol = 1e-5
-clm.Solver.Nonlinear.EtaChoice = 'Walker1'
+clm.Solver.Nonlinear.EtaChoice = "Walker1"
 clm.Solver.Nonlinear.EtaValue = 0.01
 clm.Solver.Nonlinear.UseJacobian = True
 clm.Solver.Nonlinear.DerivativeEpsilon = 1e-12
 clm.Solver.Nonlinear.StepTol = 1e-30
-clm.Solver.Nonlinear.Globalization = 'LineSearch'
+clm.Solver.Nonlinear.Globalization = "LineSearch"
 clm.Solver.Linear.KrylovDimension = 100
 clm.Solver.Linear.MaxRestarts = 5
 
-clm.Solver.Linear.Preconditioner.PCMatrixType = 'FullJacobian'
+clm.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
 
-clm.Solver.Linear.Preconditioner = 'PFMG'
+clm.Solver.Linear.Preconditioner = "PFMG"
 clm.Solver.PrintSubsurf = False
-clm.Solver.Drop = 1E-20
-clm.Solver.AbsTol = 1E-9
+clm.Solver.Drop = 1e-20
+clm.Solver.AbsTol = 1e-9
 
-clm.Solver.LSM = 'CLM'
+clm.Solver.LSM = "CLM"
 clm.Solver.WriteSiloCLM = True
-clm.Solver.CLM.MetForcing = '1D'
-clm.Solver.CLM.MetFileName = 'forcing_1.txt'
-clm.Solver.CLM.MetFilePath = './'
+clm.Solver.CLM.MetForcing = "1D"
+clm.Solver.CLM.MetFileName = "forcing_1.txt"
+clm.Solver.CLM.MetFilePath = "./"
 
-clm.Solver.CLM.EvapBeta = 'Linear'
+clm.Solver.CLM.EvapBeta = "Linear"
 
 clm.Solver.PrintSubsurfData = True
 clm.Solver.PrintPressure = False
@@ -324,7 +333,7 @@
 
 clm.Solver.PrintLSMSink = False
 clm.Solver.CLM.CLMDumpInterval = 1
-clm.Solver.CLM.CLMFileDir = 'output/'
+clm.Solver.CLM.CLMFileDir = "output/"
 clm.Solver.CLM.BinaryOutDir = False
 clm.Solver.CLM.IstepStart = 1
 clm.Solver.WriteCLMBinary = False
@@ -335,27 +344,110 @@
 clm.Solver.CLM.DailyRST = False
 clm.Solver.CLM.SingleFile = True
 
-clm.Solver.CLM.EvapBeta = 'Linear'
-clm.Solver.CLM.VegWaterStress = 'Saturation'
+clm.Solver.CLM.EvapBeta = "Linear"
+clm.Solver.CLM.VegWaterStress = "Saturation"
 clm.Solver.CLM.ResSat = 0.2
 clm.Solver.CLM.WiltingPoint = 0.2
 clm.Solver.CLM.FieldCapacity = 1.00
-clm.Solver.CLM.IrrigationType = 'none'
+clm.Solver.CLM.IrrigationType = "none"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.ICPressure.Type = 'HydroStaticPatch'
-clm.ICPressure.GeomNames = 'domain'
+clm.ICPressure.Type = "HydroStaticPatch"
+clm.ICPressure.GeomNames = "domain"
 clm.Geom.domain.ICPressure.Value = -1.0
-clm.Geom.domain.ICPressure.RefGeom = 'domain'
-clm.Geom.domain.ICPressure.RefPatch = 'z_upper'
+clm.Geom.domain.ICPressure.RefGeom = "domain"
+clm.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-for reuseCount in reuseValues:
-  new_name = f'clm_ts_{reuseCount}'
-  new_name = clm.clone(f'{new_name}')
-  new_name.Solver.CLM.ReuseCount = reuseCount
-  new_name.TimeStep.Value = (1.0 / reuseCount)
 
-  new_name.run(working_directory=dir_name)
+# Run each of the cases
+for reuseCount in reuseValues:
+    clm.Solver.CLM.ReuseCount = reuseCount
+    clm.TimeStep.Value = 1.0 / reuseCount
+
+    new_dir_name = "clm-reuse-ts-{:.2f}".format(clm.TimeStep.Value)
+    print(f"Running: {new_dir_name}")
+
+    # Run and Unload the ParFlow output files
+    clm.run(working_directory=dir_name)
+
+    for k in range(1, stopt + 1):
+        outfile1 = "{}.out.clm_output.{:05d}.C.pfb".format(run_name, k)
+
+    rm(get_absolute_path(new_dir_name))
+    mkdir(new_dir_name)
+    new_dir_path = get_absolute_path(new_dir_name)
+    os.system(
+        "bash -c 'mv {}* {}'".format(os.path.join(dir_name, run_name), new_dir_path)
+    )
+    file_path = os.path.join(dir_name, "CLM.out.clm.log")
+    os.system("mv {} {}".format(file_path, new_dir_path))
+    file_path = os.path.join(dir_name, "clm.rst.00000.0")
+    os.system("mv {} {}".format(file_path, new_dir_path))
+
+# Dictionary to store norm values for each reuseCount
+norm = {}
+file_path = {}
+ds = {}
+
+# Post process output
+with open("swe.out.csv", "w") as sweFile:
+    sweFile.write("Time")
+    for reuseCount in reuseValues:
+        norm[reuseCount] = 0.0
+        timeStep = 1.0 / reuseCount
+        sweFile.write(",{:.10e}".format(timeStep))
+    sweFile.write("\n")
+
+    compareReuse = reuseValues[0]
+
+    for k in range(1, stopt + 1):
+        sweFile.write("{}".format(k))
+
+        for reuseCount in reuseValues:
+            timeStep = 1.0 / reuseCount
+            dirname1 = get_absolute_path("clm-reuse-ts-{:.2f}".format(timeStep))
+            file_path[reuseCount] = "{}/{}.out.clm_output.{:05d}.C.pfb".format(
+                dirname1, run_name, k
+            )
+            ds[reuseCount] = read_pfb(file_path[reuseCount], z_first=False)
+
+        sweFile.write("{:d}".format(k))
+
+        for reuseCount in reuseValues:
+            if reuseCount == compareReuse:
+                norm[compareReuse] = (
+                    norm.get(compareReuse)
+                    + ds[compareReuse][0, 0, 10] * ds[compareReuse][0, 0, 10]
+                )
+            else:
+                norm[reuseCount] = norm.get(reuseCount) + (
+                    ds[compareReuse][0, 0, 10] - ds[reuseCount][0, 0, 10]
+                ) * (ds[compareReuse][0, 0, 10] - ds[reuseCount][0, 0, 10])
+
+        sweFile.write("\n")
+
+# Calculate the square root of the norms
+for reuseCount in reuseValues:
+    norm[reuseCount] = math.sqrt(norm[reuseCount])
+
+# Tests
+passed = True
+
+for reuseCount in reuseValues[1:]:
+    relerror = norm[reuseCount] / norm[compareReuse]
+    if relerror > relativeErrorTolerance:
+        print(
+            "FAILED: Relative error for reuse count = {} exceeds error tolerance ({} > {})".format(
+                reuseCount, relerror, relativeErrorTolerance
+            )
+        )
+        passed = False
+
+if passed:
+    print("clm-reuse: PASSED")
+else:
+    print("clm-reuse: FAILED")
+    sys.exit(1)
diff --git a/test/python/clm/clm.jac.py b/test/python/clm/clm.jac.py
index f77b91771..f2109444d 100644
--- a/test/python/clm/clm.jac.py
+++ b/test/python/clm/clm.jac.py
@@ -1,322 +1,406 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # this runs CLM test case
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
+import sys, argparse
 from parflow import Run
-from parflow.tools.fs import mkdir, cp, get_absolute_path
+from parflow.tools.fs import mkdir, cp, get_absolute_path, rm
+from parflow.tools.io import read_pfb, write_pfb
+from parflow.tools.compare import pf_test_file
+from parflow.tools.top import compute_top, extract_top
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Making output directories and copying input files
-#-----------------------------------------------------------------------------
-
-clm_jac = Run("clm_jac", __file__)
-
-dir_name = get_absolute_path('test_output/clm_jac')
-mkdir(dir_name)
-
-directories = ['qflx_evap_grnd', 'eflx_lh_tot', 'qflx_evap_tot', 'qflx_tran_veg', 'correct_output',
-               'qflx_infl', 'swe_out', 'eflx_lwrad_out', 't_grnd', 'diag_out', 'qflx_evap_soi', 'eflx_soil_grnd',
-               'eflx_sh_tot', 'qflx_evap_veg', 'qflx_top_soil']
+# -----------------------------------------------------------------------------
+
+run_name = "clm"
+clm_jac = Run(run_name, __file__)
+
+new_output_dir_name = get_absolute_path("test_output/clm_jac")
+mkdir(new_output_dir_name)
+
+directories = [
+    "qflx_evap_grnd",
+    "eflx_lh_tot",
+    "qflx_evap_tot",
+    "qflx_tran_veg",
+    "correct_output",
+    "qflx_infl",
+    "swe_out",
+    "eflx_lwrad_out",
+    "t_grnd",
+    "diag_out",
+    "qflx_evap_soi",
+    "eflx_soil_grnd",
+    "eflx_sh_tot",
+    "qflx_evap_veg",
+    "qflx_top_soil",
+]
 
 for directory in directories:
-    mkdir(dir_name + '/' + directory)
+    mkdir(new_output_dir_name + "/" + directory)
 
-cp('$PF_SRC/test/tcl/clm/drv_clmin.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/drv_vegm.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/drv_vegp.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/narr_1hr.sc3.txt.0', dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_clmin.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_vegm.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_vegp.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/narr_1hr.sc3.txt.0", new_output_dir_name)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # File input version number
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_jac.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Process Topology
-#-----------------------------------------------------------------------------
-
-clm_jac.Process.Topology.P = 1
-clm_jac.Process.Topology.Q = 1
-clm_jac.Process.Topology.R =1
-
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+parser = argparse.ArgumentParser()
+parser.add_argument("-p", "--p", default=1)
+parser.add_argument("-q", "--q", default=1)
+parser.add_argument("-r", "--r", default=1)
+args = parser.parse_args()
+
+clm_jac.Process.Topology.P = args.p
+clm_jac.Process.Topology.Q = args.q
+clm_jac.Process.Topology.R = args.r
+
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_jac.ComputationalGrid.Lower.X = 0.0
 clm_jac.ComputationalGrid.Lower.Y = 0.0
 clm_jac.ComputationalGrid.Lower.Z = 0.0
 
-clm_jac.ComputationalGrid.DX = 1000.
-clm_jac.ComputationalGrid.DY = 1000.
+clm_jac.ComputationalGrid.DX = 1000.0
+clm_jac.ComputationalGrid.DY = 1000.0
 clm_jac.ComputationalGrid.DZ = 0.5
 
 clm_jac.ComputationalGrid.NX = 5
 clm_jac.ComputationalGrid.NY = 5
 clm_jac.ComputationalGrid.NZ = 10
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # The Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_jac.GeomInput.Names = 'domain_input'
+clm_jac.GeomInput.Names = "domain_input"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_jac.GeomInput.domain_input.InputType = 'Box'
-clm_jac.GeomInput.domain_input.GeomName = 'domain'
+clm_jac.GeomInput.domain_input.InputType = "Box"
+clm_jac.GeomInput.domain_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_jac.Geom.domain.Lower.X = 0.0
 clm_jac.Geom.domain.Lower.Y = 0.0
 clm_jac.Geom.domain.Lower.Z = 0.0
 
-clm_jac.Geom.domain.Upper.X = 5000.
-clm_jac.Geom.domain.Upper.Y = 5000.
-clm_jac.Geom.domain.Upper.Z = 5.
+clm_jac.Geom.domain.Upper.X = 5000.0
+clm_jac.Geom.domain.Upper.Y = 5000.0
+clm_jac.Geom.domain.Upper.Z = 5.0
 
-clm_jac.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+clm_jac.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_jac.Geom.Perm.Names = 'domain'
+clm_jac.Geom.Perm.Names = "domain"
 
-clm_jac.Geom.domain.Perm.Type = 'Constant'
+clm_jac.Geom.domain.Perm.Type = "Constant"
 clm_jac.Geom.domain.Perm.Value = 0.2
 
-clm_jac.Perm.TensorType = 'TensorByGeom'
+clm_jac.Perm.TensorType = "TensorByGeom"
 
-clm_jac.Geom.Perm.TensorByGeom.Names = 'domain'
+clm_jac.Geom.Perm.TensorByGeom.Names = "domain"
 
 clm_jac.Geom.domain.Perm.TensorValX = 1.0
 clm_jac.Geom.domain.Perm.TensorValY = 1.0
 clm_jac.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # specific storage does not figure into the impes (fully sat) case but we still
 # need a key for it
 
-clm_jac.SpecificStorage.Type = 'Constant'
-clm_jac.SpecificStorage.GeomNames = 'domain'
+clm_jac.SpecificStorage.Type = "Constant"
+clm_jac.SpecificStorage.GeomNames = "domain"
 clm_jac.Geom.domain.SpecificStorage.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_jac.Phase.Names = 'water'
+clm_jac.Phase.Names = "water"
 
-clm_jac.Phase.water.Density.Type = 'Constant'
+clm_jac.Phase.water.Density.Type = "Constant"
 clm_jac.Phase.water.Density.Value = 1.0
 
-clm_jac.Phase.water.Viscosity.Type = 'Constant'
+clm_jac.Phase.water.Viscosity.Type = "Constant"
 clm_jac.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_jac.Contaminants.Names = ''
+clm_jac.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_jac.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_jac.TimingInfo.BaseUnit = 1.0
 clm_jac.TimingInfo.StartCount = 0
 clm_jac.TimingInfo.StartTime = 0.0
 clm_jac.TimingInfo.StopTime = 5
 clm_jac.TimingInfo.DumpInterval = -1
-clm_jac.TimeStep.Type = 'Constant'
+clm_jac.TimeStep.Type = "Constant"
 clm_jac.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_jac.Geom.Porosity.GeomNames = 'domain'
-clm_jac.Geom.domain.Porosity.Type = 'Constant'
+clm_jac.Geom.Porosity.GeomNames = "domain"
+clm_jac.Geom.domain.Porosity.Type = "Constant"
 clm_jac.Geom.domain.Porosity.Value = 0.390
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_jac.Domain.GeomName = 'domain'
+clm_jac.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mobility
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_jac.Phase.water.Mobility.Type = 'Constant'
+clm_jac.Phase.water.Mobility.Type = "Constant"
 clm_jac.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_jac.Phase.RelPerm.Type = 'VanGenuchten'
-clm_jac.Phase.RelPerm.GeomNames = 'domain'
+clm_jac.Phase.RelPerm.Type = "VanGenuchten"
+clm_jac.Phase.RelPerm.GeomNames = "domain"
 
 clm_jac.Geom.domain.RelPerm.Alpha = 3.5
-clm_jac.Geom.domain.RelPerm.N = 2.
+clm_jac.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm_jac.Phase.Saturation.Type = 'VanGenuchten'
-clm_jac.Phase.Saturation.GeomNames = 'domain'
+clm_jac.Phase.Saturation.Type = "VanGenuchten"
+clm_jac.Phase.Saturation.GeomNames = "domain"
 
 clm_jac.Geom.domain.Saturation.Alpha = 3.5
-clm_jac.Geom.domain.Saturation.N = 2.
+clm_jac.Geom.domain.Saturation.N = 2.0
 clm_jac.Geom.domain.Saturation.SRes = 0.01
 clm_jac.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_jac.Wells.Names = ''
+clm_jac.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_jac.Cycle.Names = 'constant'
-clm_jac.Cycle.constant.Names = 'alltime'
+clm_jac.Cycle.Names = "constant"
+clm_jac.Cycle.constant.Names = "alltime"
 clm_jac.Cycle.constant.alltime.Length = 1
 clm_jac.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_jac.BCPressure.PatchNames = clm_jac.Geom.domain.Patches
 
-clm_jac.Patch.x_lower.BCPressure.Type = 'FluxConst'
-clm_jac.Patch.x_lower.BCPressure.Cycle = 'constant'
+clm_jac.Patch.x_lower.BCPressure.Type = "FluxConst"
+clm_jac.Patch.x_lower.BCPressure.Cycle = "constant"
 clm_jac.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-clm_jac.Patch.y_lower.BCPressure.Type = 'FluxConst'
-clm_jac.Patch.y_lower.BCPressure.Cycle = 'constant'
+clm_jac.Patch.y_lower.BCPressure.Type = "FluxConst"
+clm_jac.Patch.y_lower.BCPressure.Cycle = "constant"
 clm_jac.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-clm_jac.Patch.z_lower.BCPressure.Type = 'FluxConst'
-clm_jac.Patch.z_lower.BCPressure.Cycle = 'constant'
+clm_jac.Patch.z_lower.BCPressure.Type = "FluxConst"
+clm_jac.Patch.z_lower.BCPressure.Cycle = "constant"
 clm_jac.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-clm_jac.Patch.x_upper.BCPressure.Type = 'FluxConst'
-clm_jac.Patch.x_upper.BCPressure.Cycle = 'constant'
+clm_jac.Patch.x_upper.BCPressure.Type = "FluxConst"
+clm_jac.Patch.x_upper.BCPressure.Cycle = "constant"
 clm_jac.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-clm_jac.Patch.y_upper.BCPressure.Type = 'FluxConst'
-clm_jac.Patch.y_upper.BCPressure.Cycle = 'constant'
+clm_jac.Patch.y_upper.BCPressure.Type = "FluxConst"
+clm_jac.Patch.y_upper.BCPressure.Cycle = "constant"
 clm_jac.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-clm_jac.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-clm_jac.Patch.z_upper.BCPressure.Cycle = 'constant'
+clm_jac.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+clm_jac.Patch.z_upper.BCPressure.Cycle = "constant"
 clm_jac.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm_jac.TopoSlopesX.Type = 'Constant'
-clm_jac.TopoSlopesX.GeomNames = 'domain'
+clm_jac.TopoSlopesX.Type = "Constant"
+clm_jac.TopoSlopesX.GeomNames = "domain"
 clm_jac.TopoSlopesX.Geom.domain.Value = -0.001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm_jac.TopoSlopesY.Type = 'Constant'
-clm_jac.TopoSlopesY.GeomNames = 'domain'
+clm_jac.TopoSlopesY.Type = "Constant"
+clm_jac.TopoSlopesY.GeomNames = "domain"
 clm_jac.TopoSlopesY.Geom.domain.Value = 0.001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm_jac.Mannings.Type = 'Constant'
-clm_jac.Mannings.GeomNames = 'domain'
+clm_jac.Mannings.Type = "Constant"
+clm_jac.Mannings.GeomNames = "domain"
 clm_jac.Mannings.Geom.domain.Value = 5.52e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_jac.PhaseSources.water.Type = 'Constant'
-clm_jac.PhaseSources.water.GeomNames = 'domain'
+clm_jac.PhaseSources.water.Type = "Constant"
+clm_jac.PhaseSources.water.GeomNames = "domain"
 clm_jac.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_jac.KnownSolution = 'NoKnownSolution'
+clm_jac.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_jac.Solver = 'Richards'
+clm_jac.Solver = "Richards"
 clm_jac.Solver.MaxIter = 500
 
 clm_jac.Solver.Nonlinear.MaxIter = 15
 clm_jac.Solver.Nonlinear.ResidualTol = 1e-9
-clm_jac.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+clm_jac.Solver.Nonlinear.EtaChoice = "EtaConstant"
 clm_jac.Solver.Nonlinear.EtaValue = 0.01
 clm_jac.Solver.Nonlinear.UseJacobian = True
 clm_jac.Solver.Nonlinear.DerivativeEpsilon = 1e-12
 clm_jac.Solver.Nonlinear.StepTol = 1e-20
-clm_jac.Solver.Nonlinear.Globalization = 'LineSearch'
+clm_jac.Solver.Nonlinear.Globalization = "LineSearch"
 clm_jac.Solver.Linear.KrylovDimension = 15
 clm_jac.Solver.Linear.MaxRestart = 2
 
-clm_jac.Solver.Linear.Preconditioner = 'PFMG'
+clm_jac.Solver.Linear.Preconditioner = "PFMG"
 clm_jac.Solver.PrintSubsurf = False
-clm_jac.Solver.Drop = 1E-20
-clm_jac.Solver.AbsTol = 1E-9
+clm_jac.Solver.Drop = 1e-20
+clm_jac.Solver.AbsTol = 1e-9
 
-clm_jac.Solver.LSM = 'CLM'
-clm_jac.Solver.WriteSiloCLM = True
-clm_jac.Solver.CLM.MetForcing = '1D'
-clm_jac.Solver.CLM.MetFileName = 'narr_1hr.sc3.txt.0'
-clm_jac.Solver.CLM.MetFilePath = '.'
+clm_jac.Solver.LSM = "CLM"
+clm_jac.Solver.CLM.MetForcing = "1D"
+clm_jac.Solver.CLM.MetFileName = "narr_1hr.sc3.txt.0"
+clm_jac.Solver.CLM.MetFilePath = "."
 
-clm_jac.Solver.WriteSiloEvapTrans = True
-clm_jac.Solver.WriteSiloOverlandBCFlux = True
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm_jac.ICPressure.Type = 'HydroStaticPatch'
-clm_jac.ICPressure.GeomNames = 'domain'
+clm_jac.ICPressure.Type = "HydroStaticPatch"
+clm_jac.ICPressure.GeomNames = "domain"
 clm_jac.Geom.domain.ICPressure.Value = -2.0
 
-clm_jac.Geom.domain.ICPressure.RefGeom = 'domain'
-clm_jac.Geom.domain.ICPressure.RefPatch = 'z_upper'
+clm_jac.Geom.domain.ICPressure.RefGeom = "domain"
+clm_jac.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-clm_jac.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+correct_output_dir_name = get_absolute_path("../../correct_output/clm_output")
+clm_jac.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+for i in range(6):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+mask = read_pfb(new_output_dir_name + "/clm.out.mask.pfb")
+top = compute_top(mask)
+write_pfb(new_output_dir_name + "/clm.out.top_index.pfb", top)
+
+data = read_pfb(new_output_dir_name + "/clm.out.press.00000.pfb")
+top_data = extract_top(data, top)
+write_pfb(new_output_dir_name + "/clm.out.top.press.00000.pfb", top_data)
+
+
+filename = "/clm.out.top_index.pfb"
+if not pf_test_file(
+    new_output_dir_name + filename,
+    correct_output_dir_name + filename,
+    f"Max difference in top_index",
+):
+    passed = False
+
+filename = "/clm.out.top.press.00000.pfb"
+if not pf_test_file(
+    new_output_dir_name + filename,
+    correct_output_dir_name + filename,
+    f"Max difference in top_clm.out.press.00000.pfb",
+):
+    passed = False
+
+rm(new_output_dir_name)
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/clm/clm.py b/test/python/clm/clm.py
index f8a2510b8..301674d2e 100644
--- a/test/python/clm/clm.py
+++ b/test/python/clm/clm.py
@@ -1,336 +1,406 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # this runs CLM test case
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
+import sys, argparse
 from parflow import Run
-from parflow.tools.fs import mkdir, cp, get_absolute_path
+from parflow.tools.fs import mkdir, cp, get_absolute_path, rm
+from parflow.tools.io import read_pfb, write_pfb
+from parflow.tools.compare import pf_test_file
+from parflow.tools.top import compute_top, extract_top
 
-clm = Run("clm", __file__)
+run_name = "clm"
+clm = Run(run_name, __file__)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Making output directories and copying input files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/clm')
-mkdir(dir_name)
+new_output_dir_name = get_absolute_path("test_output/clm")
+mkdir(new_output_dir_name)
 
 directories = [
-  'qflx_evap_grnd',
-  'eflx_lh_tot',
-  'qflx_evap_tot',
-  'qflx_tran_veg',
-  'correct_output',
-  'qflx_infl',
-  'swe_out',
-  'eflx_lwrad_out',
-  't_grnd',
-  'diag_out',
-  'qflx_evap_soi',
-  'eflx_soil_grnd',
-  'eflx_sh_tot',
-  'qflx_evap_veg',
-  'qflx_top_soil'
+    "qflx_evap_grnd",
+    "eflx_lh_tot",
+    "qflx_evap_tot",
+    "qflx_tran_veg",
+    "correct_output",
+    "qflx_infl",
+    "swe_out",
+    "eflx_lwrad_out",
+    "t_grnd",
+    "diag_out",
+    "qflx_evap_soi",
+    "eflx_soil_grnd",
+    "eflx_sh_tot",
+    "qflx_evap_veg",
+    "qflx_top_soil",
 ]
 
 for directory in directories:
-    mkdir(dir_name + '/' + directory)
+    mkdir(new_output_dir_name + "/" + directory)
 
-cp('$PF_SRC/test/tcl/clm/drv_clmin.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/drv_vegm.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/drv_vegp.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/narr_1hr.sc3.txt.0', dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_clmin.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_vegm.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_vegp.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/narr_1hr.sc3.txt.0", new_output_dir_name)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # File input version number
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Process Topology
-#-----------------------------------------------------------------------------
-
-clm.Process.Topology.P = 1
-clm.Process.Topology.Q = 1
-clm.Process.Topology.R = 1
-
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+parser = argparse.ArgumentParser()
+parser.add_argument("-p", "--p", default=1)
+parser.add_argument("-q", "--q", default=1)
+parser.add_argument("-r", "--r", default=1)
+args = parser.parse_args()
+
+clm.Process.Topology.P = args.p
+clm.Process.Topology.Q = args.q
+clm.Process.Topology.R = args.r
+
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.ComputationalGrid.Lower.X = 0.0
 clm.ComputationalGrid.Lower.Y = 0.0
 clm.ComputationalGrid.Lower.Z = 0.0
 
-clm.ComputationalGrid.DX = 1000.
-clm.ComputationalGrid.DY = 1000.
+clm.ComputationalGrid.DX = 1000.0
+clm.ComputationalGrid.DY = 1000.0
 clm.ComputationalGrid.DZ = 0.5
 
 clm.ComputationalGrid.NX = 5
 clm.ComputationalGrid.NY = 5
 clm.ComputationalGrid.NZ = 10
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # The Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.GeomInput.Names = 'domain_input'
+clm.GeomInput.Names = "domain_input"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.GeomInput.domain_input.InputType = 'Box'
-clm.GeomInput.domain_input.GeomName = 'domain'
+clm.GeomInput.domain_input.InputType = "Box"
+clm.GeomInput.domain_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.Geom.domain.Lower.X = 0.0
 clm.Geom.domain.Lower.Y = 0.0
 clm.Geom.domain.Lower.Z = 0.0
 
-clm.Geom.domain.Upper.X = 5000.
-clm.Geom.domain.Upper.Y = 5000.
-clm.Geom.domain.Upper.Z = 5.
+clm.Geom.domain.Upper.X = 5000.0
+clm.Geom.domain.Upper.Y = 5000.0
+clm.Geom.domain.Upper.Z = 5.0
 
-clm.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+clm.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Geom.Perm.Names = 'domain'
+clm.Geom.Perm.Names = "domain"
 
-clm.Geom.domain.Perm.Type = 'Constant'
+clm.Geom.domain.Perm.Type = "Constant"
 clm.Geom.domain.Perm.Value = 0.2
 
-clm.Perm.TensorType = 'TensorByGeom'
+clm.Perm.TensorType = "TensorByGeom"
 
-clm.Geom.Perm.TensorByGeom.Names = 'domain'
+clm.Geom.Perm.TensorByGeom.Names = "domain"
 
 clm.Geom.domain.Perm.TensorValX = 1.0
 clm.Geom.domain.Perm.TensorValY = 1.0
 clm.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # specific storage does not figure into the impes (fully sat) case but we still
 # need a key for it
 
-clm.SpecificStorage.Type = 'Constant'
-clm.SpecificStorage.GeomNames = 'domain'
+clm.SpecificStorage.Type = "Constant"
+clm.SpecificStorage.GeomNames = "domain"
 clm.Geom.domain.SpecificStorage.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Phase.Names = 'water'
+clm.Phase.Names = "water"
 
-clm.Phase.water.Density.Type = 'Constant'
+clm.Phase.water.Density.Type = "Constant"
 clm.Phase.water.Density.Value = 1.0
 
-clm.Phase.water.Viscosity.Type = 'Constant'
+clm.Phase.water.Viscosity.Type = "Constant"
 clm.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Contaminants.Names = ''
+clm.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.TimingInfo.BaseUnit = 1.0
 clm.TimingInfo.StartCount = 0
 clm.TimingInfo.StartTime = 0.0
 clm.TimingInfo.StopTime = 5
 clm.TimingInfo.DumpInterval = -1
-clm.TimeStep.Type = 'Constant'
+clm.TimeStep.Type = "Constant"
 clm.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Geom.Porosity.GeomNames = 'domain'
-clm.Geom.domain.Porosity.Type = 'Constant'
+clm.Geom.Porosity.GeomNames = "domain"
+clm.Geom.domain.Porosity.Type = "Constant"
 clm.Geom.domain.Porosity.Value = 0.390
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Domain.GeomName = 'domain'
+clm.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mobility
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Phase.water.Mobility.Type = 'Constant'
+clm.Phase.water.Mobility.Type = "Constant"
 clm.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Phase.RelPerm.Type = 'VanGenuchten'
-clm.Phase.RelPerm.GeomNames = 'domain'
+clm.Phase.RelPerm.Type = "VanGenuchten"
+clm.Phase.RelPerm.GeomNames = "domain"
 
 clm.Geom.domain.RelPerm.Alpha = 3.5
-clm.Geom.domain.RelPerm.N = 2.
+clm.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.Phase.Saturation.Type = 'VanGenuchten'
-clm.Phase.Saturation.GeomNames = 'domain'
+clm.Phase.Saturation.Type = "VanGenuchten"
+clm.Phase.Saturation.GeomNames = "domain"
 
 clm.Geom.domain.Saturation.Alpha = 3.5
-clm.Geom.domain.Saturation.N = 2.
+clm.Geom.domain.Saturation.N = 2.0
 clm.Geom.domain.Saturation.SRes = 0.01
 clm.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Wells.Names = ''
+clm.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Cycle.Names = 'constant'
-clm.Cycle.constant.Names = 'alltime'
+clm.Cycle.Names = "constant"
+clm.Cycle.constant.Names = "alltime"
 clm.Cycle.constant.alltime.Length = 1
 clm.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.BCPressure.PatchNames = clm.Geom.domain.Patches
 
-clm.Patch.x_lower.BCPressure.Type = 'FluxConst'
-clm.Patch.x_lower.BCPressure.Cycle = 'constant'
+clm.Patch.x_lower.BCPressure.Type = "FluxConst"
+clm.Patch.x_lower.BCPressure.Cycle = "constant"
 clm.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-clm.Patch.y_lower.BCPressure.Type = 'FluxConst'
-clm.Patch.y_lower.BCPressure.Cycle = 'constant'
+clm.Patch.y_lower.BCPressure.Type = "FluxConst"
+clm.Patch.y_lower.BCPressure.Cycle = "constant"
 clm.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-clm.Patch.z_lower.BCPressure.Type = 'FluxConst'
-clm.Patch.z_lower.BCPressure.Cycle = 'constant'
+clm.Patch.z_lower.BCPressure.Type = "FluxConst"
+clm.Patch.z_lower.BCPressure.Cycle = "constant"
 clm.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-clm.Patch.x_upper.BCPressure.Type = 'FluxConst'
-clm.Patch.x_upper.BCPressure.Cycle = 'constant'
+clm.Patch.x_upper.BCPressure.Type = "FluxConst"
+clm.Patch.x_upper.BCPressure.Cycle = "constant"
 clm.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-clm.Patch.y_upper.BCPressure.Type = 'FluxConst'
-clm.Patch.y_upper.BCPressure.Cycle = 'constant'
+clm.Patch.y_upper.BCPressure.Type = "FluxConst"
+clm.Patch.y_upper.BCPressure.Cycle = "constant"
 clm.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-clm.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-clm.Patch.z_upper.BCPressure.Cycle = 'constant'
+clm.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+clm.Patch.z_upper.BCPressure.Cycle = "constant"
 clm.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.TopoSlopesX.Type = 'Constant'
-clm.TopoSlopesX.GeomNames = 'domain'
+clm.TopoSlopesX.Type = "Constant"
+clm.TopoSlopesX.GeomNames = "domain"
 clm.TopoSlopesX.Geom.domain.Value = -0.001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.TopoSlopesY.Type = 'Constant'
-clm.TopoSlopesY.GeomNames = 'domain'
+clm.TopoSlopesY.Type = "Constant"
+clm.TopoSlopesY.GeomNames = "domain"
 clm.TopoSlopesY.Geom.domain.Value = 0.001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.Mannings.Type = 'Constant'
-clm.Mannings.GeomNames = 'domain'
+clm.Mannings.Type = "Constant"
+clm.Mannings.GeomNames = "domain"
 clm.Mannings.Geom.domain.Value = 5.52e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.PhaseSources.water.Type = 'Constant'
-clm.PhaseSources.water.GeomNames = 'domain'
+clm.PhaseSources.water.Type = "Constant"
+clm.PhaseSources.water.GeomNames = "domain"
 clm.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.KnownSolution = 'NoKnownSolution'
+clm.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Solver = 'Richards'
+clm.Solver = "Richards"
 clm.Solver.MaxIter = 500
 
 clm.Solver.Nonlinear.MaxIter = 15
 clm.Solver.Nonlinear.ResidualTol = 1e-9
-clm.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+clm.Solver.Nonlinear.EtaChoice = "EtaConstant"
 clm.Solver.Nonlinear.EtaValue = 0.01
 clm.Solver.Nonlinear.UseJacobian = True
 clm.Solver.Nonlinear.StepTol = 1e-20
-clm.Solver.Nonlinear.Globalization = 'LineSearch'
+clm.Solver.Nonlinear.Globalization = "LineSearch"
 clm.Solver.Linear.KrylovDimension = 15
 clm.Solver.Linear.MaxRestart = 2
 
-clm.Solver.Linear.Preconditioner = 'PFMG'
+clm.Solver.Linear.Preconditioner = "PFMG"
 clm.Solver.PrintSubsurf = False
-clm.Solver.Drop = 1E-20
-clm.Solver.AbsTol = 1E-9
+clm.Solver.Drop = 1e-20
+clm.Solver.AbsTol = 1e-9
 
-clm.Solver.LSM = 'CLM'
-clm.Solver.WriteSiloCLM = True
-clm.Solver.CLM.MetForcing = '1D'
-clm.Solver.CLM.MetFileName = 'narr_1hr.sc3.txt.0'
-clm.Solver.CLM.MetFilePath = '.'
+clm.Solver.LSM = "CLM"
+clm.Solver.CLM.MetForcing = "1D"
+clm.Solver.CLM.MetFileName = "narr_1hr.sc3.txt.0"
+clm.Solver.CLM.MetFilePath = "."
 
-clm.Solver.WriteSiloEvapTrans = True
-clm.Solver.WriteSiloOverlandBCFlux = True
 clm.Solver.PrintCLM = True
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.ICPressure.Type = 'HydroStaticPatch'
-clm.ICPressure.GeomNames = 'domain'
+clm.ICPressure.Type = "HydroStaticPatch"
+clm.ICPressure.GeomNames = "domain"
 clm.Geom.domain.ICPressure.Value = -2.0
 
-clm.Geom.domain.ICPressure.RefGeom = 'domain'
-clm.Geom.domain.ICPressure.RefPatch = 'z_upper'
+clm.Geom.domain.ICPressure.RefGeom = "domain"
+clm.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-clm.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+correct_output_dir_name = get_absolute_path("../../correct_output/clm_output")
+clm.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+for i in range(6):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+mask = read_pfb(new_output_dir_name + "/clm.out.mask.pfb")
+top = compute_top(mask)
+write_pfb(new_output_dir_name + "/clm.out.top_index.pfb", top)
+
+data = read_pfb(new_output_dir_name + "/clm.out.press.00000.pfb")
+top_data = extract_top(data, top)
+write_pfb(new_output_dir_name + "/clm.out.top.press.00000.pfb", top_data)
+
+
+filename = "/clm.out.top_index.pfb"
+if not pf_test_file(
+    new_output_dir_name + filename,
+    correct_output_dir_name + filename,
+    f"Max difference in top_index",
+):
+    passed = False
+
+filename = "/clm.out.top.press.00000.pfb"
+if not pf_test_file(
+    new_output_dir_name + filename,
+    correct_output_dir_name + filename,
+    f"Max difference in top_clm.out.press.00000.pfb",
+):
+    passed = False
+
+rm(new_output_dir_name)
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/clm/clm_4levels.py b/test/python/clm/clm_4levels.py
index 99b13ee4c..a87960c5e 100644
--- a/test/python/clm/clm_4levels.py
+++ b/test/python/clm/clm_4levels.py
@@ -1,323 +1,407 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # this runs CLM test case
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
+import sys, argparse
 from parflow import Run
-from parflow.tools.fs import mkdir, cp, get_absolute_path
+from parflow.tools.fs import mkdir, cp, get_absolute_path, rm
+from parflow.tools.io import read_pfb, write_pfb
+from parflow.tools.compare import pf_test_file
+from parflow.tools.top import compute_top, extract_top
 
-clm = Run("clm_4levels", __file__)
+run_name = "clm_4levels"
+clm = Run(run_name, __file__)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Making output directories and copying input files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/clm_4')
-mkdir(dir_name)
-
-directories = ['qflx_evap_grnd', 'eflx_lh_tot', 'qflx_evap_tot', 'qflx_tran_veg', 'correct_output',
-               'qflx_infl', 'swe_out', 'eflx_lwrad_out', 't_grnd', 'diag_out', 'qflx_evap_soi', 'eflx_soil_grnd',
-               'eflx_sh_tot', 'qflx_evap_veg', 'qflx_top_soil']
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/clm_4levels")
+mkdir(new_output_dir_name)
+
+directories = [
+    "qflx_evap_grnd",
+    "eflx_lh_tot",
+    "qflx_evap_tot",
+    "qflx_tran_veg",
+    "correct_output",
+    "qflx_infl",
+    "swe_out",
+    "eflx_lwrad_out",
+    "t_grnd",
+    "diag_out",
+    "qflx_evap_soi",
+    "eflx_soil_grnd",
+    "eflx_sh_tot",
+    "qflx_evap_veg",
+    "qflx_top_soil",
+]
 
 for directory in directories:
-    mkdir(dir_name + '/' + directory)
+    mkdir(new_output_dir_name + "/" + directory)
 
-cp('$PF_SRC/test/tcl/clm/drv_clmin.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/drv_vegm.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/drv_vegp.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/narr_1hr.sc3.txt.0', dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_clmin.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_vegm.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_vegp.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/narr_1hr.sc3.txt.0", new_output_dir_name)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # File input version number
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Process Topology
-#-----------------------------------------------------------------------------
-
-clm.Process.Topology.P = 1
-clm.Process.Topology.Q = 1
-clm.Process.Topology.R = 1
-
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+parser = argparse.ArgumentParser()
+parser.add_argument("-p", "--p", default=1)
+parser.add_argument("-q", "--q", default=1)
+parser.add_argument("-r", "--r", default=1)
+args = parser.parse_args()
+
+clm.Process.Topology.P = args.p
+clm.Process.Topology.Q = args.q
+clm.Process.Topology.R = args.r
+
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.ComputationalGrid.Lower.X = 0.0
 clm.ComputationalGrid.Lower.Y = 0.0
 clm.ComputationalGrid.Lower.Z = 0.0
 
-clm.ComputationalGrid.DX = 1000.
-clm.ComputationalGrid.DY = 1000.
+clm.ComputationalGrid.DX = 1000.0
+clm.ComputationalGrid.DY = 1000.0
 clm.ComputationalGrid.DZ = 0.5
 
 clm.ComputationalGrid.NX = 5
 clm.ComputationalGrid.NY = 5
 clm.ComputationalGrid.NZ = 10
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # The Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.GeomInput.Names = 'domain_input'
+clm.GeomInput.Names = "domain_input"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.GeomInput.domain_input.InputType = 'Box'
-clm.GeomInput.domain_input.GeomName = 'domain'
+clm.GeomInput.domain_input.InputType = "Box"
+clm.GeomInput.domain_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.Geom.domain.Lower.X = 0.0
 clm.Geom.domain.Lower.Y = 0.0
 clm.Geom.domain.Lower.Z = 0.0
 
-clm.Geom.domain.Upper.X = 5000.
-clm.Geom.domain.Upper.Y = 5000.
-clm.Geom.domain.Upper.Z = 5.
+clm.Geom.domain.Upper.X = 5000.0
+clm.Geom.domain.Upper.Y = 5000.0
+clm.Geom.domain.Upper.Z = 5.0
 
-clm.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+clm.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Geom.Perm.Names = 'domain'
+clm.Geom.Perm.Names = "domain"
 
-clm.Geom.domain.Perm.Type = 'Constant'
+clm.Geom.domain.Perm.Type = "Constant"
 clm.Geom.domain.Perm.Value = 0.2
 
-clm.Perm.TensorType = 'TensorByGeom'
+clm.Perm.TensorType = "TensorByGeom"
 
-clm.Geom.Perm.TensorByGeom.Names = 'domain'
+clm.Geom.Perm.TensorByGeom.Names = "domain"
 
 clm.Geom.domain.Perm.TensorValX = 1.0
 clm.Geom.domain.Perm.TensorValY = 1.0
 clm.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # specific storage does not figure into the impes (fully sat) case but we still
 # need a key for it
 
-clm.SpecificStorage.Type = 'Constant'
-clm.SpecificStorage.GeomNames = 'domain'
+clm.SpecificStorage.Type = "Constant"
+clm.SpecificStorage.GeomNames = "domain"
 clm.Geom.domain.SpecificStorage.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Phase.Names = 'water'
+clm.Phase.Names = "water"
 
-clm.Phase.water.Density.Type = 'Constant'
+clm.Phase.water.Density.Type = "Constant"
 clm.Phase.water.Density.Value = 1.0
 
-clm.Phase.water.Viscosity.Type = 'Constant'
+clm.Phase.water.Viscosity.Type = "Constant"
 clm.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Contaminants.Names = ''
+clm.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.TimingInfo.BaseUnit = 1.0
 clm.TimingInfo.StartCount = 0
 clm.TimingInfo.StartTime = 0.0
 clm.TimingInfo.StopTime = 5
 clm.TimingInfo.DumpInterval = -1
-clm.TimeStep.Type = 'Constant'
+clm.TimeStep.Type = "Constant"
 clm.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Geom.Porosity.GeomNames = 'domain'
-clm.Geom.domain.Porosity.Type = 'Constant'
+clm.Geom.Porosity.GeomNames = "domain"
+clm.Geom.domain.Porosity.Type = "Constant"
 clm.Geom.domain.Porosity.Value = 0.390
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Domain.GeomName = 'domain'
+clm.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mobility
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Phase.water.Mobility.Type = 'Constant'
+clm.Phase.water.Mobility.Type = "Constant"
 clm.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Phase.RelPerm.Type = 'VanGenuchten'
-clm.Phase.RelPerm.GeomNames = 'domain'
+clm.Phase.RelPerm.Type = "VanGenuchten"
+clm.Phase.RelPerm.GeomNames = "domain"
 
 clm.Geom.domain.RelPerm.Alpha = 3.5
-clm.Geom.domain.RelPerm.N = 2.
+clm.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.Phase.Saturation.Type = 'VanGenuchten'
-clm.Phase.Saturation.GeomNames = 'domain'
+clm.Phase.Saturation.Type = "VanGenuchten"
+clm.Phase.Saturation.GeomNames = "domain"
 
 clm.Geom.domain.Saturation.Alpha = 3.5
-clm.Geom.domain.Saturation.N = 2.
+clm.Geom.domain.Saturation.N = 2.0
 clm.Geom.domain.Saturation.SRes = 0.01
 clm.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Wells.Names = ''
+clm.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Cycle.Names = 'constant'
-clm.Cycle.constant.Names = 'alltime'
+clm.Cycle.Names = "constant"
+clm.Cycle.constant.Names = "alltime"
 clm.Cycle.constant.alltime.Length = 1
 clm.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 clm.BCPressure.PatchNames = clm.Geom.domain.Patches
 
-clm.Patch.x_lower.BCPressure.Type = 'FluxConst'
-clm.Patch.x_lower.BCPressure.Cycle = 'constant'
+clm.Patch.x_lower.BCPressure.Type = "FluxConst"
+clm.Patch.x_lower.BCPressure.Cycle = "constant"
 clm.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-clm.Patch.y_lower.BCPressure.Type = 'FluxConst'
-clm.Patch.y_lower.BCPressure.Cycle = 'constant'
+clm.Patch.y_lower.BCPressure.Type = "FluxConst"
+clm.Patch.y_lower.BCPressure.Cycle = "constant"
 clm.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-clm.Patch.z_lower.BCPressure.Type = 'FluxConst'
-clm.Patch.z_lower.BCPressure.Cycle = 'constant'
+clm.Patch.z_lower.BCPressure.Type = "FluxConst"
+clm.Patch.z_lower.BCPressure.Cycle = "constant"
 clm.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-clm.Patch.x_upper.BCPressure.Type = 'FluxConst'
-clm.Patch.x_upper.BCPressure.Cycle = 'constant'
+clm.Patch.x_upper.BCPressure.Type = "FluxConst"
+clm.Patch.x_upper.BCPressure.Cycle = "constant"
 clm.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-clm.Patch.y_upper.BCPressure.Type = 'FluxConst'
-clm.Patch.y_upper.BCPressure.Cycle = 'constant'
+clm.Patch.y_upper.BCPressure.Type = "FluxConst"
+clm.Patch.y_upper.BCPressure.Cycle = "constant"
 clm.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-clm.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-clm.Patch.z_upper.BCPressure.Cycle = 'constant'
+clm.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+clm.Patch.z_upper.BCPressure.Cycle = "constant"
 clm.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.TopoSlopesX.Type = 'Constant'
-clm.TopoSlopesX.GeomNames = 'domain'
+clm.TopoSlopesX.Type = "Constant"
+clm.TopoSlopesX.GeomNames = "domain"
 clm.TopoSlopesX.Geom.domain.Value = -0.001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.TopoSlopesY.Type = 'Constant'
-clm.TopoSlopesY.GeomNames = 'domain'
+clm.TopoSlopesY.Type = "Constant"
+clm.TopoSlopesY.GeomNames = "domain"
 clm.TopoSlopesY.Geom.domain.Value = 0.001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.Mannings.Type = 'Constant'
-clm.Mannings.GeomNames = 'domain'
+clm.Mannings.Type = "Constant"
+clm.Mannings.GeomNames = "domain"
 clm.Mannings.Geom.domain.Value = 5.52e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.PhaseSources.water.Type = 'Constant'
-clm.PhaseSources.water.GeomNames = 'domain'
+clm.PhaseSources.water.Type = "Constant"
+clm.PhaseSources.water.GeomNames = "domain"
 clm.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.KnownSolution = 'NoKnownSolution'
+clm.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Solver = 'Richards'
+clm.Solver = "Richards"
 clm.Solver.MaxIter = 500
 
 clm.Solver.Nonlinear.MaxIter = 15
 clm.Solver.Nonlinear.ResidualTol = 1e-9
-clm.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+clm.Solver.Nonlinear.EtaChoice = "EtaConstant"
 clm.Solver.Nonlinear.EtaValue = 0.01
 clm.Solver.Nonlinear.UseJacobian = True
 clm.Solver.Nonlinear.StepTol = 1e-20
-clm.Solver.Nonlinear.Globalization = 'LineSearch'
+clm.Solver.Nonlinear.Globalization = "LineSearch"
 clm.Solver.Linear.KrylovDimension = 15
 clm.Solver.Linear.MaxRestart = 2
 
-clm.Solver.Linear.Preconditioner = 'PFMG'
+clm.Solver.Linear.Preconditioner = "PFMG"
 clm.Solver.PrintSubsurf = False
-clm.Solver.Drop = 1E-20
-clm.Solver.AbsTol = 1E-9
+clm.Solver.Drop = 1e-20
+clm.Solver.AbsTol = 1e-9
 
-clm.Solver.LSM = 'CLM'
-clm.Solver.WriteSiloCLM = True
-clm.Solver.CLM.MetForcing = '1D'
-clm.Solver.CLM.MetFileName = 'narr_1hr.sc3.txt.0'
-clm.Solver.CLM.MetFilePath = '.'
+clm.Solver.LSM = "CLM"
+clm.Solver.CLM.MetForcing = "1D"
+clm.Solver.CLM.MetFileName = "narr_1hr.sc3.txt.0"
+clm.Solver.CLM.MetFilePath = "."
 
 clm.Solver.CLM.RootZoneNZ = 4
 
-clm.Solver.WriteSiloEvapTrans = True
-clm.Solver.WriteSiloOverlandBCFlux = True
 clm.Solver.PrintCLM = True
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.ICPressure.Type = 'HydroStaticPatch'
-clm.ICPressure.GeomNames = 'domain'
+clm.ICPressure.Type = "HydroStaticPatch"
+clm.ICPressure.GeomNames = "domain"
 clm.Geom.domain.ICPressure.Value = -2.0
 
-clm.Geom.domain.ICPressure.RefGeom = 'domain'
-clm.Geom.domain.ICPressure.RefPatch = 'z_upper'
+clm.Geom.domain.ICPressure.RefGeom = "domain"
+clm.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-clm.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+correct_output_dir_name = get_absolute_path("../../correct_output/clm_output")
+clm.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+for i in range(6):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+mask = read_pfb(new_output_dir_name + f"/{run_name}.out.mask.pfb")
+top = compute_top(mask)
+write_pfb(new_output_dir_name + f"/{run_name}.out.top_index.pfb", top)
+
+data = read_pfb(new_output_dir_name + f"/{run_name}.out.press.00000.pfb")
+top_data = extract_top(data, top)
+write_pfb(new_output_dir_name + f"/{run_name}.out.top.press.00000.pfb", top_data)
+
+
+filename = f"/{run_name}.out.top_index.pfb"
+if not pf_test_file(
+    new_output_dir_name + filename,
+    correct_output_dir_name + filename,
+    f"Max difference in top_index",
+):
+    passed = False
+
+filename = f"/{run_name}.out.top.press.00000.pfb"
+if not pf_test_file(
+    new_output_dir_name + filename,
+    correct_output_dir_name + filename,
+    f"Max difference in top_clm.out.press.00000.pfb",
+):
+    passed = False
+
+rm(new_output_dir_name)
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/clm/clm_forc_veg.py b/test/python/clm/clm_forc_veg.py
index 10009503b..f3bfbef79 100644
--- a/test/python/clm/clm_forc_veg.py
+++ b/test/python/clm/clm_forc_veg.py
@@ -1,349 +1,418 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # BH: this runs CLM with forced vegetation test case
 # lai.dat, sai.dat, displa.dat and z0m.dat are required input text files.
 # also veg_map.pfb must be provided.
 # This tests 1D forcing of vegetation: a single time series is applied at each cell based on the cell's vegetation type (IGBP)
 # These time series are stored as columns (as many columns as IGBP vegetation classes -18) in input files.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
+import sys, argparse
 from parflow import Run
-from parflow.tools.fs import mkdir, cp, chdir, get_absolute_path
+from parflow.tools.fs import mkdir, cp, chdir, get_absolute_path, rm
+from parflow.tools.io import read_pfb, write_pfb
+from parflow.tools.compare import pf_test_file
+from parflow.tools.top import compute_top, extract_top
 
-clm_veg = Run("clm_forfc_veg", __file__)
+run_name = "clm_forc_veg"
+clm_veg = Run(run_name, __file__)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Making output directories and copying input files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/clm_veg')
-mkdir(dir_name)
-chdir(dir_name)
+new_output_dir_name = get_absolute_path("test_output/clm_forc_veg")
+mkdir(new_output_dir_name)
 
 directories = [
-  'qflx_evap_grnd',
-  'eflx_lh_tot',
-  'qflx_evap_tot',
-  'qflx_tran_veg',
-  'correct_output',
-  'qflx_infl',
-  'swe_out',
-  'eflx_lwrad_out',
-  't_grnd',
-  'diag_out',
-  'qflx_evap_soi',
-  'eflx_soil_grnd',
-  'eflx_sh_tot',
-  'qflx_evap_veg',
-  'qflx_top_soil'
+    "qflx_evap_grnd",
+    "eflx_lh_tot",
+    "qflx_evap_tot",
+    "qflx_tran_veg",
+    "correct_output",
+    "qflx_infl",
+    "swe_out",
+    "eflx_lwrad_out",
+    "t_grnd",
+    "diag_out",
+    "qflx_evap_soi",
+    "eflx_soil_grnd",
+    "eflx_sh_tot",
+    "qflx_evap_veg",
+    "qflx_top_soil",
 ]
 
 for directory in directories:
-    mkdir(directory)
-
-cp('$PF_SRC/test/tcl/clm/drv_clmin.dat')
-cp('$PF_SRC/test/tcl/clm/drv_vegm.dat')
-cp('$PF_SRC/test/tcl/clm/drv_vegp.dat')
-cp('$PF_SRC/test/tcl/clm/lai.dat')
-cp('$PF_SRC/test/tcl/clm/sai.dat')
-cp('$PF_SRC/test/tcl/clm/z0m.dat')
-cp('$PF_SRC/test/tcl/clm/displa.dat')
-cp('$PF_SRC/test/tcl/clm/narr_1hr.sc3.txt.0')
-cp('$PF_SRC/test/tcl/clm/veg_map.cpfb', 'veg_map.pfb')
-
-#-----------------------------------------------------------------------------
+    mkdir(new_output_dir_name + "/" + directory)
+
+cp("$PF_SRC/test/tcl/clm/drv_clmin.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_vegm.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_vegp.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/lai.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/sai.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/z0m.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/displa.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/narr_1hr.sc3.txt.0", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/veg_map.cpfb", new_output_dir_name + "/veg_map.pfb")
+
+# -----------------------------------------------------------------------------
 # File input version number
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_veg.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Process Topology
-#-----------------------------------------------------------------------------
-
-clm_veg.Process.Topology.P = 1
-clm_veg.Process.Topology.Q = 1
-clm_veg.Process.Topology.R = 1
-
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+parser = argparse.ArgumentParser()
+parser.add_argument("-p", "--p", default=1)
+parser.add_argument("-q", "--q", default=1)
+parser.add_argument("-r", "--r", default=1)
+args = parser.parse_args()
+
+clm_veg.Process.Topology.P = args.p
+clm_veg.Process.Topology.Q = args.q
+clm_veg.Process.Topology.R = args.r
+
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 clm_veg.ComputationalGrid.Lower.X = 0.0
 clm_veg.ComputationalGrid.Lower.Y = 0.0
 clm_veg.ComputationalGrid.Lower.Z = 0.0
 
-clm_veg.ComputationalGrid.DX = 1000.
-clm_veg.ComputationalGrid.DY = 1000.
+clm_veg.ComputationalGrid.DX = 1000.0
+clm_veg.ComputationalGrid.DY = 1000.0
 clm_veg.ComputationalGrid.DZ = 0.5
 
 clm_veg.ComputationalGrid.NX = 5
 clm_veg.ComputationalGrid.NY = 5
 clm_veg.ComputationalGrid.NZ = 10
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # The Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_veg.GeomInput.Names = 'domain_input'
+clm_veg.GeomInput.Names = "domain_input"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_veg.GeomInput.domain_input.InputType = 'Box'
-clm_veg.GeomInput.domain_input.GeomName = 'domain'
+clm_veg.GeomInput.domain_input.InputType = "Box"
+clm_veg.GeomInput.domain_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_veg.Geom.domain.Lower.X = 0.0
 clm_veg.Geom.domain.Lower.Y = 0.0
 clm_veg.Geom.domain.Lower.Z = 0.0
 
-clm_veg.Geom.domain.Upper.X = 5000.
-clm_veg.Geom.domain.Upper.Y = 5000.
-clm_veg.Geom.domain.Upper.Z = 5.
+clm_veg.Geom.domain.Upper.X = 5000.0
+clm_veg.Geom.domain.Upper.Y = 5000.0
+clm_veg.Geom.domain.Upper.Z = 5.0
 
-clm_veg.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+clm_veg.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_veg.Geom.Perm.Names = 'domain'
+clm_veg.Geom.Perm.Names = "domain"
 
-clm_veg.Geom.domain.Perm.Type = 'Constant'
+clm_veg.Geom.domain.Perm.Type = "Constant"
 clm_veg.Geom.domain.Perm.Value = 0.2
 
 
-clm_veg.Perm.TensorType = 'TensorByGeom'
+clm_veg.Perm.TensorType = "TensorByGeom"
 
-clm_veg.Geom.Perm.TensorByGeom.Names = 'domain'
+clm_veg.Geom.Perm.TensorByGeom.Names = "domain"
 
 clm_veg.Geom.domain.Perm.TensorValX = 1.0
 clm_veg.Geom.domain.Perm.TensorValY = 1.0
 clm_veg.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # specific storage does not figure into the impes (fully sat) case but we still
 # need a key for it
 
-clm_veg.SpecificStorage.Type = 'Constant'
-clm_veg.SpecificStorage.GeomNames = 'domain'
+clm_veg.SpecificStorage.Type = "Constant"
+clm_veg.SpecificStorage.GeomNames = "domain"
 clm_veg.Geom.domain.SpecificStorage.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_veg.Phase.Names = 'water'
+clm_veg.Phase.Names = "water"
 
-clm_veg.Phase.water.Density.Type = 'Constant'
+clm_veg.Phase.water.Density.Type = "Constant"
 clm_veg.Phase.water.Density.Value = 1.0
 
-clm_veg.Phase.water.Viscosity.Type = 'Constant'
+clm_veg.Phase.water.Viscosity.Type = "Constant"
 clm_veg.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_veg.Contaminants.Names = ''
+clm_veg.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_veg.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_veg.TimingInfo.BaseUnit = 1.0
 clm_veg.TimingInfo.StartCount = 0
 clm_veg.TimingInfo.StartTime = 0.0
 clm_veg.TimingInfo.StopTime = 5
 clm_veg.TimingInfo.DumpInterval = -1
-clm_veg.TimeStep.Type = 'Constant'
+clm_veg.TimeStep.Type = "Constant"
 clm_veg.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_veg.Geom.Porosity.GeomNames = 'domain'
-clm_veg.Geom.domain.Porosity.Type = 'Constant'
+clm_veg.Geom.Porosity.GeomNames = "domain"
+clm_veg.Geom.domain.Porosity.Type = "Constant"
 clm_veg.Geom.domain.Porosity.Value = 0.390
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_veg.Domain.GeomName = 'domain'
+clm_veg.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mobility
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_veg.Phase.water.Mobility.Type = 'Constant'
+clm_veg.Phase.water.Mobility.Type = "Constant"
 clm_veg.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_veg.Phase.RelPerm.Type = 'VanGenuchten'
-clm_veg.Phase.RelPerm.GeomNames = 'domain'
+clm_veg.Phase.RelPerm.Type = "VanGenuchten"
+clm_veg.Phase.RelPerm.GeomNames = "domain"
 
 clm_veg.Geom.domain.RelPerm.Alpha = 3.5
-clm_veg.Geom.domain.RelPerm.N = 2.
+clm_veg.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm_veg.Phase.Saturation.Type = 'VanGenuchten'
-clm_veg.Phase.Saturation.GeomNames = 'domain'
+clm_veg.Phase.Saturation.Type = "VanGenuchten"
+clm_veg.Phase.Saturation.GeomNames = "domain"
 
 clm_veg.Geom.domain.Saturation.Alpha = 3.5
-clm_veg.Geom.domain.Saturation.N = 2.
+clm_veg.Geom.domain.Saturation.N = 2.0
 clm_veg.Geom.domain.Saturation.SRes = 0.01
 clm_veg.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_veg.Wells.Names = ''
+clm_veg.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_veg.Cycle.Names = 'constant'
-clm_veg.Cycle.constant.Names = 'alltime'
+clm_veg.Cycle.Names = "constant"
+clm_veg.Cycle.constant.Names = "alltime"
 clm_veg.Cycle.constant.alltime.Length = 1
 clm_veg.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_veg.BCPressure.PatchNames = clm_veg.Geom.domain.Patches
 
-clm_veg.Patch.x_lower.BCPressure.Type = 'FluxConst'
-clm_veg.Patch.x_lower.BCPressure.Cycle = 'constant'
+clm_veg.Patch.x_lower.BCPressure.Type = "FluxConst"
+clm_veg.Patch.x_lower.BCPressure.Cycle = "constant"
 clm_veg.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-clm_veg.Patch.y_lower.BCPressure.Type = 'FluxConst'
-clm_veg.Patch.y_lower.BCPressure.Cycle = 'constant'
+clm_veg.Patch.y_lower.BCPressure.Type = "FluxConst"
+clm_veg.Patch.y_lower.BCPressure.Cycle = "constant"
 clm_veg.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-clm_veg.Patch.z_lower.BCPressure.Type = 'FluxConst'
-clm_veg.Patch.z_lower.BCPressure.Cycle = 'constant'
+clm_veg.Patch.z_lower.BCPressure.Type = "FluxConst"
+clm_veg.Patch.z_lower.BCPressure.Cycle = "constant"
 clm_veg.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-clm_veg.Patch.x_upper.BCPressure.Type = 'FluxConst'
-clm_veg.Patch.x_upper.BCPressure.Cycle = 'constant'
+clm_veg.Patch.x_upper.BCPressure.Type = "FluxConst"
+clm_veg.Patch.x_upper.BCPressure.Cycle = "constant"
 clm_veg.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-clm_veg.Patch.y_upper.BCPressure.Type = 'FluxConst'
-clm_veg.Patch.y_upper.BCPressure.Cycle = 'constant'
+clm_veg.Patch.y_upper.BCPressure.Type = "FluxConst"
+clm_veg.Patch.y_upper.BCPressure.Cycle = "constant"
 clm_veg.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-clm_veg.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-clm_veg.Patch.z_upper.BCPressure.Cycle = 'constant'
+clm_veg.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+clm_veg.Patch.z_upper.BCPressure.Cycle = "constant"
 clm_veg.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm_veg.TopoSlopesX.Type = 'Constant'
-clm_veg.TopoSlopesX.GeomNames = 'domain'
+clm_veg.TopoSlopesX.Type = "Constant"
+clm_veg.TopoSlopesX.GeomNames = "domain"
 clm_veg.TopoSlopesX.Geom.domain.Value = -0.001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm_veg.TopoSlopesY.Type = 'Constant'
-clm_veg.TopoSlopesY.GeomNames = 'domain'
+clm_veg.TopoSlopesY.Type = "Constant"
+clm_veg.TopoSlopesY.GeomNames = "domain"
 clm_veg.TopoSlopesY.Geom.domain.Value = 0.001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm_veg.Mannings.Type = 'Constant'
-clm_veg.Mannings.GeomNames = 'domain'
+clm_veg.Mannings.Type = "Constant"
+clm_veg.Mannings.GeomNames = "domain"
 clm_veg.Mannings.Geom.domain.Value = 5.52e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_veg.PhaseSources.water.Type = 'Constant'
-clm_veg.PhaseSources.water.GeomNames = 'domain'
+clm_veg.PhaseSources.water.Type = "Constant"
+clm_veg.PhaseSources.water.GeomNames = "domain"
 clm_veg.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_veg.KnownSolution = 'NoKnownSolution'
+clm_veg.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_veg.Solver = 'Richards'
+clm_veg.Solver = "Richards"
 clm_veg.Solver.MaxIter = 500
 
 clm_veg.Solver.Nonlinear.MaxIter = 15
 clm_veg.Solver.Nonlinear.ResidualTol = 1e-9
-clm_veg.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+clm_veg.Solver.Nonlinear.EtaChoice = "EtaConstant"
 clm_veg.Solver.Nonlinear.EtaValue = 0.01
 clm_veg.Solver.Nonlinear.UseJacobian = True
 clm_veg.Solver.Nonlinear.StepTol = 1e-20
-clm_veg.Solver.Nonlinear.Globalization = 'LineSearch'
+clm_veg.Solver.Nonlinear.Globalization = "LineSearch"
 clm_veg.Solver.Linear.KrylovDimension = 15
 clm_veg.Solver.Linear.MaxRestart = 2
 
-clm_veg.Solver.Linear.Preconditioner = 'PFMG'
+clm_veg.Solver.Linear.Preconditioner = "PFMG"
 clm_veg.Solver.PrintSubsurf = False
-clm_veg.Solver.Drop = 1E-20
-clm_veg.Solver.AbsTol = 1E-9
-
-clm_veg.Solver.LSM = 'CLM'
-clm_veg.Solver.WriteSiloCLM = True
-clm_veg.Solver.CLM.MetForcing = '1D'
-clm_veg.Solver.CLM.MetFileName = 'narr_1hr.sc3.txt.0'
-clm_veg.Solver.CLM.MetFilePath = '.'
+clm_veg.Solver.Drop = 1e-20
+clm_veg.Solver.AbsTol = 1e-9
+
+clm_veg.Solver.LSM = "CLM"
+clm_veg.Solver.CLM.MetForcing = "1D"
+clm_veg.Solver.CLM.MetFileName = "narr_1hr.sc3.txt.0"
+clm_veg.Solver.CLM.MetFilePath = "."
 clm_veg.Solver.CLM.ForceVegetation = True
 
-clm_veg.Solver.WriteSiloEvapTrans = True
-clm_veg.Solver.WriteSiloOverlandBCFlux = True
 clm_veg.Solver.PrintCLM = True
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm_veg.ICPressure.Type = 'HydroStaticPatch'
-clm_veg.ICPressure.GeomNames = 'domain'
+clm_veg.ICPressure.Type = "HydroStaticPatch"
+clm_veg.ICPressure.GeomNames = "domain"
 clm_veg.Geom.domain.ICPressure.Value = -2.0
 
-clm_veg.Geom.domain.ICPressure.RefGeom = 'domain'
-clm_veg.Geom.domain.ICPressure.RefPatch = 'z_upper'
+clm_veg.Geom.domain.ICPressure.RefGeom = "domain"
+clm_veg.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-clm_veg.dist('veg_map.pfb')
-
-clm_veg.run()
+# -----------------------------------------------------------------------------
+
+clm_veg.dist(new_output_dir_name + "/veg_map.pfb")
+
+correct_output_dir_name = get_absolute_path("../../correct_output/clm_output")
+clm_veg.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+for i in range(6):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+mask = read_pfb(new_output_dir_name + f"/{run_name}.out.mask.pfb")
+top = compute_top(mask)
+write_pfb(new_output_dir_name + f"/{run_name}.out.top_index.pfb", top)
+
+data = read_pfb(new_output_dir_name + f"/{run_name}.out.press.00000.pfb")
+top_data = extract_top(data, top)
+write_pfb(new_output_dir_name + f"/{run_name}.out.top.press.00000.pfb", top_data)
+
+
+filename = f"/{run_name}.out.top_index.pfb"
+if not pf_test_file(
+    new_output_dir_name + filename,
+    correct_output_dir_name + filename,
+    f"Max difference in top_index",
+):
+    passed = False
+
+filename = f"/{run_name}.out.top.press.00000.pfb"
+if not pf_test_file(
+    new_output_dir_name + filename,
+    correct_output_dir_name + filename,
+    f"Max difference in top_clm.out.press.00000.pfb",
+):
+    passed = False
+
+rm(new_output_dir_name)
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/clm/clm_samrai.py b/test/python/clm/clm_samrai.py
index 12ba4a386..e9e2b1615 100644
--- a/test/python/clm/clm_samrai.py
+++ b/test/python/clm/clm_samrai.py
@@ -5,47 +5,48 @@
 #
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
+import sys
 
 clm_samrai = Run("clm_samrai", __file__)
 
-dir_name = get_absolute_path('test_output/clm_samrai')
+dir_name = get_absolute_path("test_output/clm_samrai")
 mkdir(dir_name)
 
 # foreach dir {qflx_evap_grnd eflx_lh_tot qflx_evap_tot qflx_tran_veg correct_output qflx_infl swe_out eflx_lwrad_out t_grnd diag_out qflx_evap_soi eflx_soil_grnd eflx_sh_tot qflx_evap_veg qflx_top_soil} {
 #     file mkdir $dir
 # }
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # File input version number
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 clm_samrai.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Process Topology
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 P = 1
 Q = 1
 R = 1
 
-NumPatches = [lindex $argv 3]
+NumPatches = sys.argv[1]
 
 clm_samrai.Process.Topology.P = P
 clm_samrai.Process.Topology.Q = Q
 clm_samrai.Process.Topology.R = R
 
-NumProcs = [expr $P * $Q * $R]
+NumProcs = P * Q * R
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_samrai.ComputationalGrid.Lower.X = 0.0
 clm_samrai.ComputationalGrid.Lower.Y = 0.0
 clm_samrai.ComputationalGrid.Lower.Z = 0.0
 
-clm_samrai.ComputationalGrid.DX = 1000.
-clm_samrai.ComputationalGrid.DY = 1000.
+clm_samrai.ComputationalGrid.DX = 1000.0
+clm_samrai.ComputationalGrid.DY = 1000.0
 clm_samrai.ComputationalGrid.DZ = 0.5
 
 clm_samrai.ComputationalGrid.NX = 5
@@ -59,7 +60,7 @@
 # 	pfset ProcessGrid.0.IX 0
 # 	pfset ProcessGrid.0.IY 0
 # 	pfset ProcessGrid.0.IZ 0
-# 	
+#
 # 	pfset ProcessGrid.0.NX 5
 # 	pfset ProcessGrid.0.NY 5
 # 	pfset ProcessGrid.0.NZ 10
@@ -69,21 +70,21 @@
 # 	pfset ProcessGrid.0.IX 0
 # 	pfset ProcessGrid.0.IY 0
 # 	pfset ProcessGrid.0.IZ 0
-# 	
+#
 # 	pfset ProcessGrid.0.NX 2
 # 	pfset ProcessGrid.0.NY 5
 # 	pfset ProcessGrid.0.NZ 10
-# 	
+#
 # 	pfset ProcessGrid.1.P 0
 # 	pfset ProcessGrid.1.IX 2
 # 	pfset ProcessGrid.1.IY 0
 # 	pfset ProcessGrid.1.IZ 0
-# 	
+#
 # 	pfset ProcessGrid.1.NX 3
 # 	pfset ProcessGrid.1.NY 5
 # 	pfset ProcessGrid.1.NZ 10
 #     } elseif {[expr $NumPatches == 3]} {
-# 	
+#
 # 	pfset ProcessGrid.NumSubgrids 3
 # 	pfset ProcessGrid.0.P 0
 # 	pfset ProcessGrid.0.IX 0
@@ -92,7 +93,7 @@
 # 	pfset ProcessGrid.0.NX 5
 # 	pfset ProcessGrid.0.NY 5
 # 	pfset ProcessGrid.0.NZ 8
-# 	
+#
 # 	pfset ProcessGrid.1.P 0
 # 	pfset ProcessGrid.1.IX 0
 # 	pfset ProcessGrid.1.IY 5
@@ -115,246 +116,246 @@
 # }
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # The Names of the GeomInputs
-#-----------------------------------------------------------------------------
-clm_samrai.GeomInput.Names = 'domain_input'
+# -----------------------------------------------------------------------------
+clm_samrai.GeomInput.Names = "domain_input"
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
-clm_samrai.GeomInput.domain_input.InputType = 'Box'
-clm_samrai.GeomInput.domain_input.GeomName = 'domain'
+# -----------------------------------------------------------------------------
+clm_samrai.GeomInput.domain_input.InputType = "Box"
+clm_samrai.GeomInput.domain_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 clm_samrai.Geom.domain.Lower.X = 0.0
 clm_samrai.Geom.domain.Lower.Y = 0.0
 clm_samrai.Geom.domain.Lower.Z = 0.0
 
-clm_samrai.Geom.domain.Upper.X = 5000.
-clm_samrai.Geom.domain.Upper.Y = 5000.
-clm_samrai.Geom.domain.Upper.Z = 5.
+clm_samrai.Geom.domain.Upper.X = 5000.0
+clm_samrai.Geom.domain.Upper.Y = 5000.0
+clm_samrai.Geom.domain.Upper.Z = 5.0
 
-clm_samrai.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+clm_samrai.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
-clm_samrai.Geom.Perm.Names = 'domain'
+# -----------------------------------------------------------------------------
+clm_samrai.Geom.Perm.Names = "domain"
 
-clm_samrai.Geom.domain.Perm.Type = 'Constant'
+clm_samrai.Geom.domain.Perm.Type = "Constant"
 clm_samrai.Geom.domain.Perm.Value = 0.2
 
 
-clm_samrai.Perm.TensorType = 'TensorByGeom'
+clm_samrai.Perm.TensorType = "TensorByGeom"
 
-clm_samrai.Geom.Perm.TensorByGeom.Names = 'domain'
+clm_samrai.Geom.Perm.TensorByGeom.Names = "domain"
 
 clm_samrai.Geom.domain.Perm.TensorValX = 1.0
 clm_samrai.Geom.domain.Perm.TensorValY = 1.0
 clm_samrai.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # specific storage does not figure into the impes (fully sat) case but we still
 # need a key for it
 
-clm_samrai.SpecificStorage.Type = 'Constant'
-clm_samrai.SpecificStorage.GeomNames = 'domain'
+clm_samrai.SpecificStorage.Type = "Constant"
+clm_samrai.SpecificStorage.GeomNames = "domain"
 clm_samrai.Geom.domain.SpecificStorage.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_samrai.Phase.Names = 'water'
+clm_samrai.Phase.Names = "water"
 
-clm_samrai.Phase.water.Density.Type = 'Constant'
+clm_samrai.Phase.water.Density.Type = "Constant"
 clm_samrai.Phase.water.Density.Value = 1.0
 
-clm_samrai.Phase.water.Viscosity.Type = 'Constant'
+clm_samrai.Phase.water.Viscosity.Type = "Constant"
 clm_samrai.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
-clm_samrai.Contaminants.Names = ''
+# -----------------------------------------------------------------------------
+clm_samrai.Contaminants.Names = ""
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_samrai.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
-#  
+# -----------------------------------------------------------------------------
+#
 clm_samrai.TimingInfo.BaseUnit = 1.0
 clm_samrai.TimingInfo.StartCount = 0
 clm_samrai.TimingInfo.StartTime = 0.0
 clm_samrai.TimingInfo.StopTime = 5
 clm_samrai.TimingInfo.DumpInterval = -1
-clm_samrai.TimeStep.Type = 'Constant'
+clm_samrai.TimeStep.Type = "Constant"
 clm_samrai.TimeStep.Value = 1.0
-#  
+#
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_samrai.Geom.Porosity.GeomNames = 'domain'
+clm_samrai.Geom.Porosity.GeomNames = "domain"
 
-clm_samrai.Geom.domain.Porosity.Type = 'Constant'
+clm_samrai.Geom.domain.Porosity.Type = "Constant"
 clm_samrai.Geom.domain.Porosity.Value = 0.390
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
-clm_samrai.Domain.GeomName = 'domain'
+# -----------------------------------------------------------------------------
+clm_samrai.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mobility
-#-----------------------------------------------------------------------------
-clm_samrai.Phase.water.Mobility.Type = 'Constant'
+# -----------------------------------------------------------------------------
+clm_samrai.Phase.water.Mobility.Type = "Constant"
 clm_samrai.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
-#  
-clm_samrai.Phase.RelPerm.Type = 'VanGenuchten'
-clm_samrai.Phase.RelPerm.GeomNames = 'domain'
-#  
+# -----------------------------------------------------------------------------
+#
+clm_samrai.Phase.RelPerm.Type = "VanGenuchten"
+clm_samrai.Phase.RelPerm.GeomNames = "domain"
+#
 clm_samrai.Geom.domain.RelPerm.Alpha = 3.5
-clm_samrai.Geom.domain.RelPerm.N = 2.
+clm_samrai.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm_samrai.Phase.Saturation.Type = 'VanGenuchten'
-clm_samrai.Phase.Saturation.GeomNames = 'domain'
-#  
+clm_samrai.Phase.Saturation.Type = "VanGenuchten"
+clm_samrai.Phase.Saturation.GeomNames = "domain"
+#
 clm_samrai.Geom.domain.Saturation.Alpha = 3.5
-clm_samrai.Geom.domain.Saturation.N = 2.
+clm_samrai.Geom.domain.Saturation.N = 2.0
 clm_samrai.Geom.domain.Saturation.SRes = 0.01
 clm_samrai.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
-clm_samrai.Wells.Names = ''
+# -----------------------------------------------------------------------------
+clm_samrai.Wells.Names = ""
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
-clm_samrai.Cycle.Names = 'constant'
-clm_samrai.Cycle.constant.Names = 'alltime'
+# -----------------------------------------------------------------------------
+clm_samrai.Cycle.Names = "constant"
+clm_samrai.Cycle.constant.Names = "alltime"
 clm_samrai.Cycle.constant.alltime.Length = 1
 clm_samrai.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
-clm_samrai.BCPressure.PatchNames = [pfget Geom.domain.Patches]
-#  
-clm_samrai.Patch.x_lower.BCPressure.Type = 'FluxConst'
-clm_samrai.Patch.x_lower.BCPressure.Cycle = 'constant'
+# -----------------------------------------------------------------------------
+clm_samrai.BCPressure.PatchNames = clm_samrai.Geom.domain.Patches
+#
+clm_samrai.Patch.x_lower.BCPressure.Type = "FluxConst"
+clm_samrai.Patch.x_lower.BCPressure.Cycle = "constant"
 clm_samrai.Patch.x_lower.BCPressure.alltime.Value = 0.0
-#  
-clm_samrai.Patch.y_lower.BCPressure.Type = 'FluxConst'
-clm_samrai.Patch.y_lower.BCPressure.Cycle = 'constant'
+#
+clm_samrai.Patch.y_lower.BCPressure.Type = "FluxConst"
+clm_samrai.Patch.y_lower.BCPressure.Cycle = "constant"
 clm_samrai.Patch.y_lower.BCPressure.alltime.Value = 0.0
-#  
-clm_samrai.Patch.z_lower.BCPressure.Type = 'FluxConst'
-clm_samrai.Patch.z_lower.BCPressure.Cycle = 'constant'
+#
+clm_samrai.Patch.z_lower.BCPressure.Type = "FluxConst"
+clm_samrai.Patch.z_lower.BCPressure.Cycle = "constant"
 clm_samrai.Patch.z_lower.BCPressure.alltime.Value = 0.0
-#  
-clm_samrai.Patch.x_upper.BCPressure.Type = 'FluxConst'
-clm_samrai.Patch.x_upper.BCPressure.Cycle = 'constant'
+#
+clm_samrai.Patch.x_upper.BCPressure.Type = "FluxConst"
+clm_samrai.Patch.x_upper.BCPressure.Cycle = "constant"
 clm_samrai.Patch.x_upper.BCPressure.alltime.Value = 0.0
-#  
-clm_samrai.Patch.y_upper.BCPressure.Type = 'FluxConst'
-clm_samrai.Patch.y_upper.BCPressure.Cycle = 'constant'
+#
+clm_samrai.Patch.y_upper.BCPressure.Type = "FluxConst"
+clm_samrai.Patch.y_upper.BCPressure.Cycle = "constant"
 clm_samrai.Patch.y_upper.BCPressure.alltime.Value = 0.0
-#  
-clm_samrai.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-##pfset Patch.z-upper.BCPressure.Type                FluxConst 
-clm_samrai.Patch.z_upper.BCPressure.Cycle = 'constant'
+#
+clm_samrai.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+##pfset Patch.z-upper.BCPressure.Type                FluxConst
+clm_samrai.Patch.z_upper.BCPressure.Cycle = "constant"
 clm_samrai.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
-#  
-clm_samrai.TopoSlopesX.Type = 'Constant'
-clm_samrai.TopoSlopesX.GeomNames = 'domain'
+# ---------------------------------------------------------
+#
+clm_samrai.TopoSlopesX.Type = "Constant"
+clm_samrai.TopoSlopesX.GeomNames = "domain"
 clm_samrai.TopoSlopesX.Geom.domain.Value = -0.001
-#  
-#---------------------------------------------------------
+#
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
-#  
-clm_samrai.TopoSlopesY.Type = 'Constant'
-clm_samrai.TopoSlopesY.GeomNames = 'domain'
+# ---------------------------------------------------------
+#
+clm_samrai.TopoSlopesY.Type = "Constant"
+clm_samrai.TopoSlopesY.GeomNames = "domain"
 clm_samrai.TopoSlopesY.Geom.domain.Value = 0.001
-#  
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
-#  
-clm_samrai.Mannings.Type = 'Constant'
-clm_samrai.Mannings.GeomNames = 'domain'
+#
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
+#
+clm_samrai.Mannings.Type = "Constant"
+clm_samrai.Mannings.GeomNames = "domain"
 clm_samrai.Mannings.Geom.domain.Value = 5.52e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_samrai.PhaseSources.water.Type = 'Constant'
-clm_samrai.PhaseSources.water.GeomNames = 'domain'
+clm_samrai.PhaseSources.water.Type = "Constant"
+clm_samrai.PhaseSources.water.GeomNames = "domain"
 clm_samrai.PhaseSources.water.Geom.domain.Value = 0.0
-#  
-#-----------------------------------------------------------------------------
+#
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
-#  
-clm_samrai.KnownSolution = 'NoKnownSolution'
+# -----------------------------------------------------------------------------
+#
+clm_samrai.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
-#  
-clm_samrai.Solver = 'Richards'
+# -----------------------------------------------------------------------------
+#
+clm_samrai.Solver = "Richards"
 clm_samrai.Solver.MaxIter = 500
-#  
+#
 clm_samrai.Solver.Nonlinear.MaxIter = 15
 clm_samrai.Solver.Nonlinear.ResidualTol = 1e-9
-clm_samrai.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+clm_samrai.Solver.Nonlinear.EtaChoice = "EtaConstant"
 clm_samrai.Solver.Nonlinear.EtaValue = 0.01
 clm_samrai.Solver.Nonlinear.UseJacobian = False
 clm_samrai.Solver.Nonlinear.DerivativeEpsilon = 1e-12
 clm_samrai.Solver.Nonlinear.StepTol = 1e-20
-clm_samrai.Solver.Nonlinear.Globalization = 'LineSearch'
+clm_samrai.Solver.Nonlinear.Globalization = "LineSearch"
 clm_samrai.Solver.Linear.KrylovDimension = 15
 clm_samrai.Solver.Linear.MaxRestart = 2
-#  
-clm_samrai.Solver.Linear.Preconditioner = 'PFMGOctree'
+#
+clm_samrai.Solver.Linear.Preconditioner = "PFMGOctree"
 
 clm_samrai.Solver.Linear.Preconditioner.PFMGOctree.BoxSizePowerOf2 = 2
-#  
-clm_samrai.Solver.LSM = 'CLM'
+#
+clm_samrai.Solver.LSM = "CLM"
 clm_samrai.Solver.WriteSiloCLM = True
-clm_samrai.Solver.CLM.MetForcing = 1D
-clm_samrai.Solver.CLM.MetFileName = 'narr_1hr.sc3.txt.0'
-clm_samrai.Solver.CLM.MetFilePath = ./
+clm_samrai.Solver.CLM.MetForcing = "1D"
+clm_samrai.Solver.CLM.MetFileName = "narr_1hr.sc3.txt.0"
+clm_samrai.Solver.CLM.MetFilePath = "./"
 
 
 clm_samrai.Solver.WriteSiloEvapTrans = True
@@ -362,18 +363,17 @@
 
 
 # Initial conditions: water pressure
-#---------------------------------------------------------
-#  
-clm_samrai.ICPressure.Type = 'HydroStaticPatch'
-clm_samrai.ICPressure.GeomNames = 'domain'
+# ---------------------------------------------------------
+#
+clm_samrai.ICPressure.Type = "HydroStaticPatch"
+clm_samrai.ICPressure.GeomNames = "domain"
 clm_samrai.Geom.domain.ICPressure.Value = -2.0
-#  
-clm_samrai.Geom.domain.ICPressure.RefGeom = 'domain'
-clm_samrai.Geom.domain.ICPressure.RefPatch = 'z_upper'
-
+#
+clm_samrai.Geom.domain.ICPressure.RefGeom = "domain"
+clm_samrai.Geom.domain.ICPressure.RefPatch = "z_upper"
 
 
-num_processors = [expr [pfget Process.Topology.P] * [pfget Process.Topology.Q] * [pfget Process.Topology.R]]
+# num_processors = NumProcs
 # for {set i 0} { $i <= $num_processors } {incr i} {
 #     file delete drv_vegm.dat.$i
 #     file copy  drv_vegm.dat drv_vegm.dat.$i
@@ -381,16 +381,16 @@
 #     file copy drv_clmin.dat drv_clmin.dat.$i
 # }
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 
-# pfrun clm 
-# pfundist clm 
+# pfrun clm
+# pfundist clm
 
 #
-# Tests 
+# Tests
 #
 # source ../pftest.tcl
 passed = 1
@@ -415,13 +415,13 @@
 #     }
 # }
 
-mask = [pfload clm.out.mask.pfb]
-top = [Parflow::pfcomputetop $mask]
+# mask = [pfload clm.out.mask.pfb]
+# top = [Parflow::pfcomputetop $mask]
 
 # pfsave $top -pfb "clm.out.top_index.pfb"
 
-data = [pfload clm.out.press.00000.pfb]
-top_data = [Parflow::pfextracttop $top $data]
+# data = [pfload clm.out.press.00000.pfb]
+# top_data = [Parflow::pfextracttop $top $data]
 
 # pfsave $data -pfb "clm.out.press.00000.pfb"
 # pfsave $top_data -pfb "clm.out.top.press.00000.pfb"
@@ -440,7 +440,6 @@
 # }
 
 
-
 # if $passed {
 #     puts "clm : PASSED"
 # } {
diff --git a/test/python/clm/clm_slope.py b/test/python/clm/clm_slope.py
index 34b97ae95..927d79b3c 100644
--- a/test/python/clm/clm_slope.py
+++ b/test/python/clm/clm_slope.py
@@ -1,106 +1,120 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # this runs CLM_slope test case
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 from parflow import Run
 from parflow.tools.fs import mkdir, cp, get_absolute_path
 
 clm = Run("clm_slope", __file__)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Making output directories and copying input files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/clm_slopes')
+dir_name = get_absolute_path("test_output/clm_slopes")
 mkdir(dir_name)
 
-directories = ['qflx_evap_grnd', 'eflx_lh_tot', 'qflx_evap_tot', 'qflx_tran_veg', 'correct_output',
-               'qflx_infl', 'swe_out', 'eflx_lwrad_out', 't_grnd', 'diag_out', 'qflx_evap_soi', 'eflx_soil_grnd',
-               'eflx_sh_tot', 'qflx_evap_veg', 'qflx_top_soil']
+directories = [
+    "qflx_evap_grnd",
+    "eflx_lh_tot",
+    "qflx_evap_tot",
+    "qflx_tran_veg",
+    "correct_output",
+    "qflx_infl",
+    "swe_out",
+    "eflx_lwrad_out",
+    "t_grnd",
+    "diag_out",
+    "qflx_evap_soi",
+    "eflx_soil_grnd",
+    "eflx_sh_tot",
+    "qflx_evap_veg",
+    "qflx_top_soil",
+]
 
 for directory in directories:
-    mkdir(dir_name + '/' + directory)
+    mkdir(dir_name + "/" + directory)
 
-cp('$PF_SRC/test/tcl/clm/drv_clmin.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/drv_vegm.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/drv_vegp.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/narr_1hr.sc3.txt.0', dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_clmin.dat", dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_vegm.dat", dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_vegp.dat", dir_name)
+cp("$PF_SRC/test/tcl/clm/narr_1hr.sc3.txt.0", dir_name)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # File input version number
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Process Topology
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.Process.Topology.P = 1
 clm.Process.Topology.Q = 1
 clm.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 clm.ComputationalGrid.Lower.X = 0.0
 clm.ComputationalGrid.Lower.Y = 0.0
 clm.ComputationalGrid.Lower.Z = 0.0
 
-clm.ComputationalGrid.DX = 1000.
-clm.ComputationalGrid.DY = 1000.
+clm.ComputationalGrid.DX = 1000.0
+clm.ComputationalGrid.DY = 1000.0
 clm.ComputationalGrid.DZ = 0.5
 
 clm.ComputationalGrid.NX = 5
 clm.ComputationalGrid.NY = 5
 clm.ComputationalGrid.NZ = 10
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # The Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.GeomInput.Names = 'domain_input center_input north_input south_input \
-    east_input west_input northeast_input southeast_input southwest_input northwest_input'
+clm.GeomInput.Names = "domain_input center_input north_input south_input \
+    east_input west_input northeast_input southeast_input southwest_input northwest_input"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
-
-clm.GeomInput.domain_input.GeomName = 'domain'
-clm.GeomInput.center_input.GeomName = 'center'
-clm.GeomInput.north_input.GeomName = 'north'
-clm.GeomInput.south_input.GeomName = 'south'
-clm.GeomInput.east_input.GeomName = 'east'
-clm.GeomInput.west_input.GeomName = 'west'
-clm.GeomInput.northeast_input.GeomName = 'northeast'
-clm.GeomInput.southeast_input.GeomName = 'southeast'
-clm.GeomInput.southwest_input.GeomName = 'southwest'
-clm.GeomInput.northwest_input.GeomName = 'northwest'
-
-clm.GeomInput.domain_input.InputType = 'Box'
-clm.GeomInput.center_input.InputType = 'Box'
-clm.GeomInput.north_input.InputType = 'Box'
-clm.GeomInput.south_input.InputType = 'Box'
-clm.GeomInput.east_input.InputType = 'Box'
-clm.GeomInput.west_input.InputType = 'Box'
-clm.GeomInput.northeast_input.InputType = 'Box'
-clm.GeomInput.southeast_input.InputType = 'Box'
-clm.GeomInput.southwest_input.InputType = 'Box'
-clm.GeomInput.northwest_input.InputType = 'Box'
-
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+
+clm.GeomInput.domain_input.GeomName = "domain"
+clm.GeomInput.center_input.GeomName = "center"
+clm.GeomInput.north_input.GeomName = "north"
+clm.GeomInput.south_input.GeomName = "south"
+clm.GeomInput.east_input.GeomName = "east"
+clm.GeomInput.west_input.GeomName = "west"
+clm.GeomInput.northeast_input.GeomName = "northeast"
+clm.GeomInput.southeast_input.GeomName = "southeast"
+clm.GeomInput.southwest_input.GeomName = "southwest"
+clm.GeomInput.northwest_input.GeomName = "northwest"
+
+clm.GeomInput.domain_input.InputType = "Box"
+clm.GeomInput.center_input.InputType = "Box"
+clm.GeomInput.north_input.InputType = "Box"
+clm.GeomInput.south_input.InputType = "Box"
+clm.GeomInput.east_input.InputType = "Box"
+clm.GeomInput.west_input.InputType = "Box"
+clm.GeomInput.northeast_input.InputType = "Box"
+clm.GeomInput.southeast_input.InputType = "Box"
+clm.GeomInput.southwest_input.InputType = "Box"
+clm.GeomInput.northwest_input.InputType = "Box"
+
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.Geom.domain.Lower.X = 0.0
 clm.Geom.domain.Lower.Y = 0.0
 clm.Geom.domain.Lower.Z = 0.0
 
-clm.Geom.domain.Upper.X = 5000.
-clm.Geom.domain.Upper.Y = 5000.
-clm.Geom.domain.Upper.Z = 5.
+clm.Geom.domain.Upper.X = 5000.0
+clm.Geom.domain.Upper.Y = 5000.0
+clm.Geom.domain.Upper.Z = 5.0
 
-clm.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+clm.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
 # ---------------------------------------------------------
 # Center tile Geometry
@@ -210,163 +224,165 @@
 clm.Geom.northwest.Upper.Y = 5000
 clm.Geom.northwest.Upper.Z = 5.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Geom.Perm.Names = 'domain'
+clm.Geom.Perm.Names = "domain"
 
-clm.Geom.domain.Perm.Type = 'Constant'
+clm.Geom.domain.Perm.Type = "Constant"
 clm.Geom.domain.Perm.Value = 0.2
 
-clm.Perm.TensorType = 'TensorByGeom'
+clm.Perm.TensorType = "TensorByGeom"
 
-clm.Geom.Perm.TensorByGeom.Names = 'domain'
+clm.Geom.Perm.TensorByGeom.Names = "domain"
 
 clm.Geom.domain.Perm.TensorValX = 1.0
 clm.Geom.domain.Perm.TensorValY = 1.0
 clm.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # specific storage does not figure into the impes (fully sat) case but we still
 # need a key for it
 
-clm.SpecificStorage.Type = 'Constant'
-clm.SpecificStorage.GeomNames = 'domain'
+clm.SpecificStorage.Type = "Constant"
+clm.SpecificStorage.GeomNames = "domain"
 clm.Geom.domain.SpecificStorage.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Phase.Names = 'water'
+clm.Phase.Names = "water"
 
-clm.Phase.water.Density.Type = 'Constant'
+clm.Phase.water.Density.Type = "Constant"
 clm.Phase.water.Density.Value = 1.0
 
-clm.Phase.water.Viscosity.Type = 'Constant'
+clm.Phase.water.Viscosity.Type = "Constant"
 clm.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Contaminants.Names = ''
+clm.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.TimingInfo.BaseUnit = 1.0
 clm.TimingInfo.StartCount = 0
 clm.TimingInfo.StartTime = 0.0
 clm.TimingInfo.StopTime = 5
 clm.TimingInfo.DumpInterval = -1
-clm.TimeStep.Type = 'Constant'
+clm.TimeStep.Type = "Constant"
 clm.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Geom.Porosity.GeomNames = 'domain'
+clm.Geom.Porosity.GeomNames = "domain"
 
-clm.Geom.domain.Porosity.Type = 'Constant'
+clm.Geom.domain.Porosity.Type = "Constant"
 clm.Geom.domain.Porosity.Value = 0.390
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Domain.GeomName = 'domain'
+clm.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mobility
-#-----------------------------------------------------------------------------
-clm.Phase.water.Mobility.Type = 'Constant'
+# -----------------------------------------------------------------------------
+clm.Phase.water.Mobility.Type = "Constant"
 clm.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Phase.RelPerm.Type = 'VanGenuchten'
-clm.Phase.RelPerm.GeomNames = 'domain'
+clm.Phase.RelPerm.Type = "VanGenuchten"
+clm.Phase.RelPerm.GeomNames = "domain"
 
 clm.Geom.domain.RelPerm.Alpha = 3.5
-clm.Geom.domain.RelPerm.N = 2.
+clm.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.Phase.Saturation.Type = 'VanGenuchten'
-clm.Phase.Saturation.GeomNames = 'domain'
+clm.Phase.Saturation.Type = "VanGenuchten"
+clm.Phase.Saturation.GeomNames = "domain"
 
 clm.Geom.domain.Saturation.Alpha = 3.5
-clm.Geom.domain.Saturation.N = 2.
+clm.Geom.domain.Saturation.N = 2.0
 clm.Geom.domain.Saturation.SRes = 0.01
 clm.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Wells.Names = ''
+clm.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Cycle.Names = 'constant'
-clm.Cycle.constant.Names = 'alltime'
+clm.Cycle.Names = "constant"
+clm.Cycle.constant.Names = "alltime"
 clm.Cycle.constant.alltime.Length = 1
 clm.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.BCPressure.PatchNames = clm.Geom.domain.Patches
 
-clm.Patch.x_lower.BCPressure.Type = 'FluxConst'
-clm.Patch.x_lower.BCPressure.Cycle = 'constant'
+clm.Patch.x_lower.BCPressure.Type = "FluxConst"
+clm.Patch.x_lower.BCPressure.Cycle = "constant"
 clm.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-clm.Patch.y_lower.BCPressure.Type = 'FluxConst'
-clm.Patch.y_lower.BCPressure.Cycle = 'constant'
+clm.Patch.y_lower.BCPressure.Type = "FluxConst"
+clm.Patch.y_lower.BCPressure.Cycle = "constant"
 clm.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-clm.Patch.z_lower.BCPressure.Type = 'FluxConst'
-clm.Patch.z_lower.BCPressure.Cycle = 'constant'
+clm.Patch.z_lower.BCPressure.Type = "FluxConst"
+clm.Patch.z_lower.BCPressure.Cycle = "constant"
 clm.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-clm.Patch.x_upper.BCPressure.Type = 'FluxConst'
-clm.Patch.x_upper.BCPressure.Cycle = 'constant'
+clm.Patch.x_upper.BCPressure.Type = "FluxConst"
+clm.Patch.x_upper.BCPressure.Cycle = "constant"
 clm.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-clm.Patch.y_upper.BCPressure.Type = 'FluxConst'
-clm.Patch.y_upper.BCPressure.Cycle = 'constant'
+clm.Patch.y_upper.BCPressure.Type = "FluxConst"
+clm.Patch.y_upper.BCPressure.Cycle = "constant"
 clm.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-clm.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-clm.Patch.z_upper.BCPressure.Cycle = 'constant'
+clm.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+clm.Patch.z_upper.BCPressure.Cycle = "constant"
 clm.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.TopoSlopesX.Type = 'Constant'
-clm.TopoSlopesX.GeomNames = 'center north south east west northeast southeast southwest northwest'
+clm.TopoSlopesX.Type = "Constant"
+clm.TopoSlopesX.GeomNames = (
+    "center north south east west northeast southeast southwest northwest"
+)
 clm.TopoSlopesX.Geom.center.Value = 0.0
 clm.TopoSlopesX.Geom.north.Value = 0.0
 clm.TopoSlopesX.Geom.south.Value = 0.0
@@ -377,12 +393,14 @@
 clm.TopoSlopesX.Geom.southwest.Value = 10.0
 clm.TopoSlopesX.Geom.northwest.Value = 10.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.TopoSlopesY.Type = 'Constant'
-clm.TopoSlopesY.GeomNames = 'center north south east west northeast southeast southwest northwest'
+clm.TopoSlopesY.Type = "Constant"
+clm.TopoSlopesY.GeomNames = (
+    "center north south east west northeast southeast southwest northwest"
+)
 clm.TopoSlopesY.Geom.center.Value = 0.0
 clm.TopoSlopesY.Geom.north.Value = -10.0
 clm.TopoSlopesY.Geom.south.Value = 10.0
@@ -393,74 +411,74 @@
 clm.TopoSlopesY.Geom.southwest.Value = 10.0
 clm.TopoSlopesY.Geom.northwest.Value = -10.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.Mannings.Type = 'Constant'
-clm.Mannings.GeomNames = 'domain'
+clm.Mannings.Type = "Constant"
+clm.Mannings.GeomNames = "domain"
 clm.Mannings.Geom.domain.Value = 5.52e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.PhaseSources.water.Type = 'Constant'
-clm.PhaseSources.water.GeomNames = 'domain'
+clm.PhaseSources.water.Type = "Constant"
+clm.PhaseSources.water.GeomNames = "domain"
 clm.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.KnownSolution = 'NoKnownSolution'
+clm.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Solver = 'Richards'
+clm.Solver = "Richards"
 clm.Solver.MaxIter = 500
 
 clm.Solver.Nonlinear.MaxIter = 15
 clm.Solver.Nonlinear.ResidualTol = 1e-9
-clm.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+clm.Solver.Nonlinear.EtaChoice = "EtaConstant"
 clm.Solver.Nonlinear.EtaValue = 0.01
 clm.Solver.Nonlinear.UseJacobian = True
 clm.Solver.Nonlinear.StepTol = 1e-20
-clm.Solver.Nonlinear.Globalization = 'LineSearch'
+clm.Solver.Nonlinear.Globalization = "LineSearch"
 clm.Solver.Linear.KrylovDimension = 15
 clm.Solver.Linear.MaxRestart = 2
 
-clm.Solver.Linear.Preconditioner = 'PFMG'
+clm.Solver.Linear.Preconditioner = "PFMG"
 clm.Solver.PrintSubsurf = False
-clm.Solver.Drop = 1E-20
-clm.Solver.AbsTol = 1E-9
+clm.Solver.Drop = 1e-20
+clm.Solver.AbsTol = 1e-9
 
-clm.Solver.LSM = 'CLM'
+clm.Solver.LSM = "CLM"
 clm.Solver.WriteSiloCLM = False
-clm.Solver.CLM.MetForcing = '1D'
-clm.Solver.CLM.MetFileName = 'narr_1hr.sc3.txt.0'
-clm.Solver.CLM.MetFilePath = '.'
+clm.Solver.CLM.MetForcing = "1D"
+clm.Solver.CLM.MetFileName = "narr_1hr.sc3.txt.0"
+clm.Solver.CLM.MetFilePath = "."
 
 clm.Solver.WriteSiloEvapTrans = False
 clm.Solver.WriteSiloOverlandBCFlux = False
 clm.Solver.PrintCLM = False
 clm.Solver.SlopeAccountingCLM = True
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.ICPressure.Type = 'HydroStaticPatch'
-clm.ICPressure.GeomNames = 'domain'
+clm.ICPressure.Type = "HydroStaticPatch"
+clm.ICPressure.GeomNames = "domain"
 clm.Geom.domain.ICPressure.Value = -2.0
 
-clm.Geom.domain.ICPressure.RefGeom = 'domain'
-clm.Geom.domain.ICPressure.RefPatch = 'z_upper'
+clm.Geom.domain.ICPressure.RefGeom = "domain"
+clm.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.run(working_directory=dir_name)
diff --git a/test/python/clm/clm_varDZ.py b/test/python/clm/clm_varDZ.py
index 7af8cbd9c..7fd9a53bf 100644
--- a/test/python/clm/clm_varDZ.py
+++ b/test/python/clm/clm_varDZ.py
@@ -1,4 +1,4 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # BH: this runs CLM with variable DZ test case
 # It has been observed that root fraction distribution in CLM was based on constant DZ,
 # regardless of whether variable DZ was an input in PF. This is fixed in this version.
@@ -6,357 +6,427 @@
 # only center of cells location match. As a consequence, the volume of the column is not
 # conserved. This has been corrected in this version, and the root fraction distribution
 # is calculated based on layer interfaces which match ParFlow layer interfaces.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
+import sys, argparse
 from parflow import Run
-from parflow.tools.fs import cp, mkdir, get_absolute_path
+from parflow.tools.fs import cp, mkdir, get_absolute_path, rm
+from parflow.tools.io import read_pfb, write_pfb
+from parflow.tools.compare import pf_test_file
+from parflow.tools.top import compute_top, extract_top
 
-clm = Run("clm_varDZ", __file__)
+run_name = "clm_varDZ"
+clm = Run(run_name, __file__)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Making output directories and copying input files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/clm_vardz')
-mkdir(dir_name)
+new_output_dir_name = get_absolute_path("test_output/clm_vardz")
+mkdir(new_output_dir_name)
 
 directories = [
-  'qflx_evap_grnd',
-  'eflx_lh_tot',
-  'qflx_evap_tot',
-  'qflx_tran_veg',
-  'correct_output',
-  'qflx_infl',
-  'swe_out',
-  'eflx_lwrad_out',
-  't_grnd',
-  'diag_out',
-  'qflx_evap_soi',
-  'eflx_soil_grnd',
-  'eflx_sh_tot',
-  'qflx_evap_veg',
-  'qflx_top_soil'
+    "qflx_evap_grnd",
+    "eflx_lh_tot",
+    "qflx_evap_tot",
+    "qflx_tran_veg",
+    "correct_output",
+    "qflx_infl",
+    "swe_out",
+    "eflx_lwrad_out",
+    "t_grnd",
+    "diag_out",
+    "qflx_evap_soi",
+    "eflx_soil_grnd",
+    "eflx_sh_tot",
+    "qflx_evap_veg",
+    "qflx_top_soil",
 ]
 
 for directory in directories:
-    mkdir(dir_name + '/' + directory)
+    mkdir(new_output_dir_name + "/" + directory)
 
-cp('$PF_SRC/test/tcl/clm/drv_clmin.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/drv_vegm.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/drv_vegp.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/narr_1hr.sc3.txt.0', dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_clmin.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_vegm.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_vegp.dat", new_output_dir_name)
+cp("$PF_SRC/test/tcl/clm/narr_1hr.sc3.txt.0", new_output_dir_name)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # File input version number
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Process Topology
-#-----------------------------------------------------------------------------
-
-clm.Process.Topology.P = 1
-clm.Process.Topology.Q = 1
-clm.Process.Topology.R = 1
-
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+parser = argparse.ArgumentParser()
+parser.add_argument("-p", "--p", default=1)
+parser.add_argument("-q", "--q", default=1)
+parser.add_argument("-r", "--r", default=1)
+args = parser.parse_args()
+
+clm.Process.Topology.P = args.p
+clm.Process.Topology.Q = args.q
+clm.Process.Topology.R = args.r
+
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.ComputationalGrid.Lower.X = 0.0
 clm.ComputationalGrid.Lower.Y = 0.0
 clm.ComputationalGrid.Lower.Z = 0.0
 
-clm.ComputationalGrid.DX = 1000.
-clm.ComputationalGrid.DY = 1000.
+clm.ComputationalGrid.DX = 1000.0
+clm.ComputationalGrid.DY = 1000.0
 clm.ComputationalGrid.DZ = 0.5
 
 clm.ComputationalGrid.NX = 5
 clm.ComputationalGrid.NY = 5
 clm.ComputationalGrid.NZ = 10
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # The Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.GeomInput.Names = 'domain_input'
+clm.GeomInput.Names = "domain_input"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.GeomInput.domain_input.InputType = 'Box'
-clm.GeomInput.domain_input.GeomName = 'domain'
+clm.GeomInput.domain_input.InputType = "Box"
+clm.GeomInput.domain_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.Geom.domain.Lower.X = 0.0
 clm.Geom.domain.Lower.Y = 0.0
 clm.Geom.domain.Lower.Z = 0.0
 
-clm.Geom.domain.Upper.X = 5000.
-clm.Geom.domain.Upper.Y = 5000.
-clm.Geom.domain.Upper.Z = 5.
+clm.Geom.domain.Upper.X = 5000.0
+clm.Geom.domain.Upper.Y = 5000.0
+clm.Geom.domain.Upper.Z = 5.0
 
-clm.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+clm.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#--------------------------------------------
+# --------------------------------------------
 # variable dz assignments
-#------------------------------------------
+# ------------------------------------------
 
 clm.Solver.Nonlinear.VariableDz = True
-clm.dzScale.GeomNames = 'domain'
-clm.dzScale.Type = 'nzList'
+clm.dzScale.GeomNames = "domain"
+clm.dzScale.Type = "nzList"
 clm.dzScale.nzListNumber = 10
 
 clm.Cell._0.dzScale.Value = 2.5
 clm.Cell._1.dzScale.Value = 2
 clm.Cell._2.dzScale.Value = 1.5
 clm.Cell._3.dzScale.Value = 1.25
-clm.Cell._4.dzScale.Value = 1.
+clm.Cell._4.dzScale.Value = 1.0
 clm.Cell._5.dzScale.Value = 0.75
 clm.Cell._6.dzScale.Value = 0.5
 clm.Cell._7.dzScale.Value = 0.25
 clm.Cell._8.dzScale.Value = 0.125
 clm.Cell._9.dzScale.Value = 0.125
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Geom.Perm.Names = 'domain'
+clm.Geom.Perm.Names = "domain"
 
-clm.Geom.domain.Perm.Type = 'Constant'
+clm.Geom.domain.Perm.Type = "Constant"
 clm.Geom.domain.Perm.Value = 0.2
 
-clm.Perm.TensorType = 'TensorByGeom'
+clm.Perm.TensorType = "TensorByGeom"
 
-clm.Geom.Perm.TensorByGeom.Names = 'domain'
+clm.Geom.Perm.TensorByGeom.Names = "domain"
 
 clm.Geom.domain.Perm.TensorValX = 1.0
 clm.Geom.domain.Perm.TensorValY = 1.0
 clm.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # specific storage does not figure into the impes (fully sat) case but we still
 # need a key for it
 
-clm.SpecificStorage.Type = 'Constant'
-clm.SpecificStorage.GeomNames = 'domain'
+clm.SpecificStorage.Type = "Constant"
+clm.SpecificStorage.GeomNames = "domain"
 clm.Geom.domain.SpecificStorage.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Phase.Names = 'water'
+clm.Phase.Names = "water"
 
-clm.Phase.water.Density.Type = 'Constant'
+clm.Phase.water.Density.Type = "Constant"
 clm.Phase.water.Density.Value = 1.0
 
-clm.Phase.water.Viscosity.Type = 'Constant'
+clm.Phase.water.Viscosity.Type = "Constant"
 clm.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Contaminants.Names = ''
+clm.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm.TimingInfo.BaseUnit = 1.0
 clm.TimingInfo.StartCount = 0
 clm.TimingInfo.StartTime = 0.0
 clm.TimingInfo.StopTime = 5
 clm.TimingInfo.DumpInterval = -1
-clm.TimeStep.Type = 'Constant'
+clm.TimeStep.Type = "Constant"
 clm.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Geom.Porosity.GeomNames = 'domain'
-clm.Geom.domain.Porosity.Type = 'Constant'
+clm.Geom.Porosity.GeomNames = "domain"
+clm.Geom.domain.Porosity.Type = "Constant"
 clm.Geom.domain.Porosity.Value = 0.390
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Domain.GeomName = 'domain'
+clm.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mobility
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Phase.water.Mobility.Type = 'Constant'
+clm.Phase.water.Mobility.Type = "Constant"
 clm.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Phase.RelPerm.Type = 'VanGenuchten'
-clm.Phase.RelPerm.GeomNames = 'domain'
+clm.Phase.RelPerm.Type = "VanGenuchten"
+clm.Phase.RelPerm.GeomNames = "domain"
 
 clm.Geom.domain.RelPerm.Alpha = 3.5
-clm.Geom.domain.RelPerm.N = 2.
+clm.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.Phase.Saturation.Type = 'VanGenuchten'
-clm.Phase.Saturation.GeomNames = 'domain'
+clm.Phase.Saturation.Type = "VanGenuchten"
+clm.Phase.Saturation.GeomNames = "domain"
 
 clm.Geom.domain.Saturation.Alpha = 3.5
-clm.Geom.domain.Saturation.N = 2.
+clm.Geom.domain.Saturation.N = 2.0
 clm.Geom.domain.Saturation.SRes = 0.01
 clm.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Wells.Names = ''
+clm.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Cycle.Names = 'constant'
-clm.Cycle.constant.Names = 'alltime'
+clm.Cycle.Names = "constant"
+clm.Cycle.constant.Names = "alltime"
 clm.Cycle.constant.alltime.Length = 1
 clm.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 clm.BCPressure.PatchNames = clm.Geom.domain.Patches
 
-clm.Patch.x_lower.BCPressure.Type = 'FluxConst'
-clm.Patch.x_lower.BCPressure.Cycle = 'constant'
+clm.Patch.x_lower.BCPressure.Type = "FluxConst"
+clm.Patch.x_lower.BCPressure.Cycle = "constant"
 clm.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-clm.Patch.y_lower.BCPressure.Type = 'FluxConst'
-clm.Patch.y_lower.BCPressure.Cycle = 'constant'
+clm.Patch.y_lower.BCPressure.Type = "FluxConst"
+clm.Patch.y_lower.BCPressure.Cycle = "constant"
 clm.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-clm.Patch.z_lower.BCPressure.Type = 'FluxConst'
-clm.Patch.z_lower.BCPressure.Cycle = 'constant'
+clm.Patch.z_lower.BCPressure.Type = "FluxConst"
+clm.Patch.z_lower.BCPressure.Cycle = "constant"
 clm.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-clm.Patch.x_upper.BCPressure.Type = 'FluxConst'
-clm.Patch.x_upper.BCPressure.Cycle = 'constant'
+clm.Patch.x_upper.BCPressure.Type = "FluxConst"
+clm.Patch.x_upper.BCPressure.Cycle = "constant"
 clm.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-clm.Patch.y_upper.BCPressure.Type = 'FluxConst'
-clm.Patch.y_upper.BCPressure.Cycle = 'constant'
+clm.Patch.y_upper.BCPressure.Type = "FluxConst"
+clm.Patch.y_upper.BCPressure.Cycle = "constant"
 clm.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-clm.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-clm.Patch.z_upper.BCPressure.Cycle = 'constant'
+clm.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+clm.Patch.z_upper.BCPressure.Cycle = "constant"
 clm.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.TopoSlopesX.Type = 'Constant'
-clm.TopoSlopesX.GeomNames = 'domain'
+clm.TopoSlopesX.Type = "Constant"
+clm.TopoSlopesX.GeomNames = "domain"
 clm.TopoSlopesX.Geom.domain.Value = -0.001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.TopoSlopesY.Type = 'Constant'
-clm.TopoSlopesY.GeomNames = 'domain'
+clm.TopoSlopesY.Type = "Constant"
+clm.TopoSlopesY.GeomNames = "domain"
 clm.TopoSlopesY.Geom.domain.Value = 0.001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.Mannings.Type = 'Constant'
-clm.Mannings.GeomNames = 'domain'
+clm.Mannings.Type = "Constant"
+clm.Mannings.GeomNames = "domain"
 clm.Mannings.Geom.domain.Value = 5.52e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.PhaseSources.water.Type = 'Constant'
-clm.PhaseSources.water.GeomNames = 'domain'
+clm.PhaseSources.water.Type = "Constant"
+clm.PhaseSources.water.GeomNames = "domain"
 clm.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.KnownSolution = 'NoKnownSolution'
+clm.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm.Solver = 'Richards'
+clm.Solver = "Richards"
 clm.Solver.MaxIter = 500
 
 clm.Solver.Nonlinear.MaxIter = 15
 clm.Solver.Nonlinear.ResidualTol = 1e-9
-clm.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+clm.Solver.Nonlinear.EtaChoice = "EtaConstant"
 clm.Solver.Nonlinear.EtaValue = 0.01
 clm.Solver.Nonlinear.UseJacobian = True
 clm.Solver.Nonlinear.StepTol = 1e-20
-clm.Solver.Nonlinear.Globalization = 'LineSearch'
+clm.Solver.Nonlinear.Globalization = "LineSearch"
 clm.Solver.Linear.KrylovDimension = 15
 clm.Solver.Linear.MaxRestart = 2
 
-clm.Solver.Linear.Preconditioner = 'PFMG'
+clm.Solver.Linear.Preconditioner = "PFMG"
 clm.Solver.PrintSubsurf = False
-clm.Solver.Drop = 1E-20
-clm.Solver.AbsTol = 1E-9
-
-clm.Solver.LSM = 'CLM'
-clm.Solver.WriteSiloCLM = True
-clm.Solver.CLM.MetForcing = '1D'
-clm.Solver.CLM.MetFileName = 'narr_1hr.sc3.txt.0'
-clm.Solver.CLM.MetFilePath = '.'
+clm.Solver.Drop = 1e-20
+clm.Solver.AbsTol = 1e-9
+
+clm.Solver.LSM = "CLM"
+clm.Solver.CLM.MetForcing = "1D"
+clm.Solver.CLM.MetFileName = "narr_1hr.sc3.txt.0"
+clm.Solver.CLM.MetFilePath = "."
 clm.Solver.CLM.ForceVegetation = False
 
-clm.Solver.WriteSiloEvapTrans = True
-clm.Solver.WriteSiloOverlandBCFlux = True
 clm.Solver.PrintCLM = True
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.ICPressure.Type = 'HydroStaticPatch'
-clm.ICPressure.GeomNames = 'domain'
+clm.ICPressure.Type = "HydroStaticPatch"
+clm.ICPressure.GeomNames = "domain"
 clm.Geom.domain.ICPressure.Value = -2.0
 
-clm.Geom.domain.ICPressure.RefGeom = 'domain'
-clm.Geom.domain.ICPressure.RefPatch = 'z_upper'
+clm.Geom.domain.ICPressure.RefGeom = "domain"
+clm.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-clm.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+correct_output_dir_name = get_absolute_path("../../correct_output/clm_output")
+clm.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+for i in range(6):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+mask = read_pfb(new_output_dir_name + f"/{run_name}.out.mask.pfb")
+top = compute_top(mask)
+write_pfb(new_output_dir_name + f"/{run_name}.out.top_index.pfb", top)
+
+data = read_pfb(new_output_dir_name + f"/{run_name}.out.press.00000.pfb")
+top_data = extract_top(data, top)
+write_pfb(new_output_dir_name + f"/{run_name}.out.top.press.00000.pfb", top_data)
+
+
+filename = f"/{run_name}.out.top_index.pfb"
+if not pf_test_file(
+    new_output_dir_name + filename,
+    correct_output_dir_name + filename,
+    f"Max difference in top_index",
+):
+    passed = False
+
+filename = f"/{run_name}.out.top.press.00000.pfb"
+if not pf_test_file(
+    new_output_dir_name + filename,
+    correct_output_dir_name + filename,
+    f"Max difference in top_clm.out.press.00000.pfb",
+):
+    passed = False
+
+rm(new_output_dir_name)
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/clm/clm_vtk.py b/test/python/clm/clm_vtk.py
index e27d4b929..39a8d6c6d 100644
--- a/test/python/clm/clm_vtk.py
+++ b/test/python/clm/clm_vtk.py
@@ -1,98 +1,112 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # this runs CLM test case with vtk (not included in the ParFlow test suite)
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 from parflow import Run
 from parflow.tools.fs import cp, mkdir, get_absolute_path
 
 clm_vtk = Run("clm_vtk", __file__)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Making output directories and copying input files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/clm_vtk')
+dir_name = get_absolute_path("test_output/clm_vtk")
 mkdir(dir_name)
 
-directories = ['qflx_evap_grnd', 'eflx_lh_tot', 'qflx_evap_tot', 'qflx_tran_veg', 'correct_output',
-               'qflx_infl', 'swe_out', 'eflx_lwrad_out', 't_grnd', 'diag_out', 'qflx_evap_soi', 'eflx_soil_grnd',
-               'eflx_sh_tot', 'qflx_evap_veg', 'qflx_top_soil']
+directories = [
+    "qflx_evap_grnd",
+    "eflx_lh_tot",
+    "qflx_evap_tot",
+    "qflx_tran_veg",
+    "correct_output",
+    "qflx_infl",
+    "swe_out",
+    "eflx_lwrad_out",
+    "t_grnd",
+    "diag_out",
+    "qflx_evap_soi",
+    "eflx_soil_grnd",
+    "eflx_sh_tot",
+    "qflx_evap_veg",
+    "qflx_top_soil",
+]
 
 for directory in directories:
-    mkdir(dir_name + '/' + directory)
+    mkdir(dir_name + "/" + directory)
 
-cp('$PF_SRC/test/tcl/clm/drv_clmin.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/drv_vegm.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/drv_vegp.dat', dir_name)
-cp('$PF_SRC/test/tcl/clm/narr_1hr.sc3.txt.0', dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_clmin.dat", dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_vegm.dat", dir_name)
+cp("$PF_SRC/test/tcl/clm/drv_vegp.dat", dir_name)
+cp("$PF_SRC/test/tcl/clm/narr_1hr.sc3.txt.0", dir_name)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # File input version number
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_vtk.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Process Topology
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_vtk.Process.Topology.P = 1
 clm_vtk.Process.Topology.Q = 1
 clm_vtk.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_vtk.ComputationalGrid.Lower.X = 0.0
 clm_vtk.ComputationalGrid.Lower.Y = 0.0
 clm_vtk.ComputationalGrid.Lower.Z = 0.0
 
-clm_vtk.ComputationalGrid.DX = 1000.
-clm_vtk.ComputationalGrid.DY = 1000.
+clm_vtk.ComputationalGrid.DX = 1000.0
+clm_vtk.ComputationalGrid.DY = 1000.0
 clm_vtk.ComputationalGrid.DZ = 0.5
 
 clm_vtk.ComputationalGrid.NX = 5
 clm_vtk.ComputationalGrid.NY = 5
 clm_vtk.ComputationalGrid.NZ = 10
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # The Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_vtk.GeomInput.Names = 'domain_input'
+clm_vtk.GeomInput.Names = "domain_input"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_vtk.GeomInput.domain_input.InputType = 'Box'
-clm_vtk.GeomInput.domain_input.GeomName = 'domain'
+clm_vtk.GeomInput.domain_input.InputType = "Box"
+clm_vtk.GeomInput.domain_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_vtk.Geom.domain.Lower.X = 0.0
 clm_vtk.Geom.domain.Lower.Y = 0.0
 clm_vtk.Geom.domain.Lower.Z = 0.0
 
-clm_vtk.Geom.domain.Upper.X = 5000.
-clm_vtk.Geom.domain.Upper.Y = 5000.
-clm_vtk.Geom.domain.Upper.Z = 5.
+clm_vtk.Geom.domain.Upper.X = 5000.0
+clm_vtk.Geom.domain.Upper.Y = 5000.0
+clm_vtk.Geom.domain.Upper.Z = 5.0
 
-clm_vtk.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+clm_vtk.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_vtk.Geom.Perm.Names = 'domain'
+clm_vtk.Geom.Perm.Names = "domain"
 
-clm_vtk.Geom.domain.Perm.Type = 'TurnBands'
-clm_vtk.Geom.domain.Perm.LambdaX = 3000.
-clm_vtk.Geom.domain.Perm.LambdaY = 2000.
-clm_vtk.Geom.domain.Perm.LambdaZ = 5.
+clm_vtk.Geom.domain.Perm.Type = "TurnBands"
+clm_vtk.Geom.domain.Perm.LambdaX = 3000.0
+clm_vtk.Geom.domain.Perm.LambdaY = 2000.0
+clm_vtk.Geom.domain.Perm.LambdaZ = 5.0
 clm_vtk.Geom.domain.Perm.GeomMean = 0.2
 clm_vtk.Geom.domain.Perm.Sigma = 0.5
 clm_vtk.Geom.domain.Perm.NumLines = 40
@@ -100,217 +114,217 @@
 clm_vtk.Geom.domain.Perm.KMax = 100.0
 clm_vtk.Geom.domain.Perm.DelK = 0.2
 clm_vtk.Geom.domain.Perm.Seed = 23333
-clm_vtk.Geom.domain.Perm.LogNormal = 'Log'
-clm_vtk.Geom.domain.Perm.StratType = 'Bottom'
+clm_vtk.Geom.domain.Perm.LogNormal = "Log"
+clm_vtk.Geom.domain.Perm.StratType = "Bottom"
 
-clm_vtk.Perm.TensorType = 'TensorByGeom'
+clm_vtk.Perm.TensorType = "TensorByGeom"
 
-clm_vtk.Geom.Perm.TensorByGeom.Names = 'domain'
+clm_vtk.Geom.Perm.TensorByGeom.Names = "domain"
 
 clm_vtk.Geom.domain.Perm.TensorValX = 1.0
 clm_vtk.Geom.domain.Perm.TensorValY = 1.0
 clm_vtk.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # specific storage does not figure into the impes (fully sat) case but we still
 # need a key for it
 
-clm_vtk.SpecificStorage.Type = 'Constant'
-clm_vtk.SpecificStorage.GeomNames = 'domain'
+clm_vtk.SpecificStorage.Type = "Constant"
+clm_vtk.SpecificStorage.GeomNames = "domain"
 clm_vtk.Geom.domain.SpecificStorage.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_vtk.Phase.Names = 'water'
+clm_vtk.Phase.Names = "water"
 
-clm_vtk.Phase.water.Density.Type = 'Constant'
+clm_vtk.Phase.water.Density.Type = "Constant"
 clm_vtk.Phase.water.Density.Value = 1.0
 
-clm_vtk.Phase.water.Viscosity.Type = 'Constant'
+clm_vtk.Phase.water.Viscosity.Type = "Constant"
 clm_vtk.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_vtk.Contaminants.Names = ''
+clm_vtk.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_vtk.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_vtk.TimingInfo.BaseUnit = 1.0
 clm_vtk.TimingInfo.StartCount = 0
 clm_vtk.TimingInfo.StartTime = 0.0
 clm_vtk.TimingInfo.StopTime = 5
 clm_vtk.TimingInfo.DumpInterval = -1
-clm_vtk.TimeStep.Type = 'Constant'
+clm_vtk.TimeStep.Type = "Constant"
 clm_vtk.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_vtk.Geom.Porosity.GeomNames = 'domain'
+clm_vtk.Geom.Porosity.GeomNames = "domain"
 
-clm_vtk.Geom.domain.Porosity.Type = 'Constant'
+clm_vtk.Geom.domain.Porosity.Type = "Constant"
 clm_vtk.Geom.domain.Porosity.Value = 0.390
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_vtk.Domain.GeomName = 'domain'
+clm_vtk.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mobility
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_vtk.Phase.water.Mobility.Type = 'Constant'
+clm_vtk.Phase.water.Mobility.Type = "Constant"
 clm_vtk.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_vtk.Phase.RelPerm.Type = 'VanGenuchten'
-clm_vtk.Phase.RelPerm.GeomNames = 'domain'
+clm_vtk.Phase.RelPerm.Type = "VanGenuchten"
+clm_vtk.Phase.RelPerm.GeomNames = "domain"
 
 clm_vtk.Geom.domain.RelPerm.Alpha = 3.5
-clm_vtk.Geom.domain.RelPerm.N = 2.
+clm_vtk.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm_vtk.Phase.Saturation.Type = 'VanGenuchten'
-clm_vtk.Phase.Saturation.GeomNames = 'domain'
+clm_vtk.Phase.Saturation.Type = "VanGenuchten"
+clm_vtk.Phase.Saturation.GeomNames = "domain"
 
 clm_vtk.Geom.domain.Saturation.Alpha = 3.5
-clm_vtk.Geom.domain.Saturation.N = 2.
+clm_vtk.Geom.domain.Saturation.N = 2.0
 clm_vtk.Geom.domain.Saturation.SRes = 0.01
 clm_vtk.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_vtk.Wells.Names = ''
+clm_vtk.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_vtk.Cycle.Names = 'constant'
-clm_vtk.Cycle.constant.Names = 'alltime'
+clm_vtk.Cycle.Names = "constant"
+clm_vtk.Cycle.constant.Names = "alltime"
 clm_vtk.Cycle.constant.alltime.Length = 1
 clm_vtk.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_vtk.BCPressure.PatchNames = clm_vtk.Geom.domain.Patches
 
-clm_vtk.Patch.x_lower.BCPressure.Type = 'FluxConst'
-clm_vtk.Patch.x_lower.BCPressure.Cycle = 'constant'
+clm_vtk.Patch.x_lower.BCPressure.Type = "FluxConst"
+clm_vtk.Patch.x_lower.BCPressure.Cycle = "constant"
 clm_vtk.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-clm_vtk.Patch.y_lower.BCPressure.Type = 'FluxConst'
-clm_vtk.Patch.y_lower.BCPressure.Cycle = 'constant'
+clm_vtk.Patch.y_lower.BCPressure.Type = "FluxConst"
+clm_vtk.Patch.y_lower.BCPressure.Cycle = "constant"
 clm_vtk.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-clm_vtk.Patch.z_lower.BCPressure.Type = 'FluxConst'
-clm_vtk.Patch.z_lower.BCPressure.Cycle = 'constant'
+clm_vtk.Patch.z_lower.BCPressure.Type = "FluxConst"
+clm_vtk.Patch.z_lower.BCPressure.Cycle = "constant"
 clm_vtk.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-clm_vtk.Patch.x_upper.BCPressure.Type = 'FluxConst'
-clm_vtk.Patch.x_upper.BCPressure.Cycle = 'constant'
+clm_vtk.Patch.x_upper.BCPressure.Type = "FluxConst"
+clm_vtk.Patch.x_upper.BCPressure.Cycle = "constant"
 clm_vtk.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-clm_vtk.Patch.y_upper.BCPressure.Type = 'FluxConst'
-clm_vtk.Patch.y_upper.BCPressure.Cycle = 'constant'
+clm_vtk.Patch.y_upper.BCPressure.Type = "FluxConst"
+clm_vtk.Patch.y_upper.BCPressure.Cycle = "constant"
 clm_vtk.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-clm_vtk.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-clm_vtk.Patch.z_upper.BCPressure.Cycle = 'constant'
+clm_vtk.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+clm_vtk.Patch.z_upper.BCPressure.Cycle = "constant"
 clm_vtk.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm_vtk.TopoSlopesX.Type = 'Constant'
-clm_vtk.TopoSlopesX.GeomNames = 'domain'
+clm_vtk.TopoSlopesX.Type = "Constant"
+clm_vtk.TopoSlopesX.GeomNames = "domain"
 clm_vtk.TopoSlopesX.Geom.domain.Value = -0.001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm_vtk.TopoSlopesY.Type = 'Constant'
-clm_vtk.TopoSlopesY.GeomNames = 'domain'
+clm_vtk.TopoSlopesY.Type = "Constant"
+clm_vtk.TopoSlopesY.GeomNames = "domain"
 clm_vtk.TopoSlopesY.Geom.domain.Value = 0.001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm_vtk.Mannings.Type = 'Constant'
-clm_vtk.Mannings.GeomNames = 'domain'
+clm_vtk.Mannings.Type = "Constant"
+clm_vtk.Mannings.GeomNames = "domain"
 clm_vtk.Mannings.Geom.domain.Value = 5.52e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_vtk.PhaseSources.water.Type = 'Constant'
-clm_vtk.PhaseSources.water.GeomNames = 'domain'
+clm_vtk.PhaseSources.water.Type = "Constant"
+clm_vtk.PhaseSources.water.GeomNames = "domain"
 clm_vtk.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_vtk.KnownSolution = 'NoKnownSolution'
+clm_vtk.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-clm_vtk.Solver = 'Richards'
+clm_vtk.Solver = "Richards"
 clm_vtk.Solver.MaxIter = 500
 
 clm_vtk.Solver.Nonlinear.MaxIter = 75
 clm_vtk.Solver.Nonlinear.ResidualTol = 1e-9
-clm_vtk.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+clm_vtk.Solver.Nonlinear.EtaChoice = "EtaConstant"
 clm_vtk.Solver.Nonlinear.EtaValue = 0.01
 clm_vtk.Solver.Nonlinear.UseJacobian = True
 clm_vtk.Solver.Nonlinear.StepTol = 1e-20
-clm_vtk.Solver.Nonlinear.Globalization = 'LineSearch'
+clm_vtk.Solver.Nonlinear.Globalization = "LineSearch"
 clm_vtk.Solver.Linear.KrylovDimension = 15
 clm_vtk.Solver.Linear.MaxRestart = 2
 
-clm_vtk.Solver.Linear.Preconditioner = 'PFMG'
+clm_vtk.Solver.Linear.Preconditioner = "PFMG"
 clm_vtk.Solver.PrintSubsurf = False
-clm_vtk.Solver.Drop = 1E-20
-clm_vtk.Solver.AbsTol = 1E-9
+clm_vtk.Solver.Drop = 1e-20
+clm_vtk.Solver.AbsTol = 1e-9
 
-clm_vtk.Solver.LSM = 'CLM'
+clm_vtk.Solver.LSM = "CLM"
 clm_vtk.Solver.WriteSiloCLM = False
-clm_vtk.Solver.CLM.MetForcing = '1D'
-clm_vtk.Solver.CLM.MetFileName = 'narr_1hr.sc3.txt.0'
-clm_vtk.Solver.CLM.MetFilePath = '.'
+clm_vtk.Solver.CLM.MetForcing = "1D"
+clm_vtk.Solver.CLM.MetFileName = "narr_1hr.sc3.txt.0"
+clm_vtk.Solver.CLM.MetFilePath = "."
 
 clm_vtk.Solver.WriteSiloEvapTrans = False
 clm_vtk.Solver.WriteSiloOverlandBCFlux = False
@@ -322,26 +336,26 @@
 clm_vtk.Solver.WriteSiloCLM = False
 
 clm_vtk.Solver.PrintLSMSink = False
-clm_vtk.Solver.CLM.CLMFileDir = './'
+clm_vtk.Solver.CLM.CLMFileDir = "./"
 clm_vtk.Solver.CLM.BinaryOutDir = False
 
 clm_vtk.Solver.CLM.WriteLastRST = True
 clm_vtk.Solver.CLM.WriteLogs = False
 clm_vtk.Solver.CLM.DailyRST = False
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm_vtk.ICPressure.Type = 'HydroStaticPatch'
-clm_vtk.ICPressure.GeomNames = 'domain'
+clm_vtk.ICPressure.Type = "HydroStaticPatch"
+clm_vtk.ICPressure.GeomNames = "domain"
 clm_vtk.Geom.domain.ICPressure.Value = -2.0
 
-clm_vtk.Geom.domain.ICPressure.RefGeom = 'domain'
-clm_vtk.Geom.domain.ICPressure.RefPatch = 'z_upper'
+clm_vtk.Geom.domain.ICPressure.RefGeom = "domain"
+clm_vtk.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 clm_vtk.run(working_directory=dir_name)
diff --git a/test/python/crater2D.py b/test/python/crater2D.py
index 3f5f1332d..27d3c1bea 100644
--- a/test/python/crater2D.py
+++ b/test/python/crater2D.py
@@ -1,20 +1,23 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 #  This is a 2D crater problem w/ time varying input and topography
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import cp, mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
-crater = Run("crater", __file__)
+run_name = "crater"
+crater = Run(run_name, __file__)
 
 # ---------------------------------------------------------
 # Copy testing data in test directory
 # ---------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/crater')
+dir_name = get_absolute_path("test_output/crater")
 mkdir(dir_name)
 
-cp('$PF_SRC/test/input/crater2D.pfsol', dir_name)
+cp("$PF_SRC/test/input/crater2D.pfsol", dir_name)
 
 # ---------------------------------------------------------
 
@@ -24,9 +27,9 @@
 crater.Process.Topology.Q = 1
 crater.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 crater.ComputationalGrid.Lower.X = 0.0
 crater.ComputationalGrid.Lower.Y = 0.0
@@ -52,20 +55,20 @@
 crater.ComputationalGrid.DY = (UpperY - LowerY) / NY
 crater.ComputationalGrid.DZ = (UpperZ - LowerZ) / NZ
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 Zones = "zone1 zone2 zone3above4 zone3left4 zone3right4 zone3below4 zone4"
 
-crater.GeomInput.Names = f'solidinput {Zones} background'
+crater.GeomInput.Names = f"solidinput {Zones} background"
 
-crater.GeomInput.solidinput.InputType = 'SolidFile'
-crater.GeomInput.solidinput.GeomNames = 'domain'
-crater.GeomInput.solidinput.FileName = 'crater2D.pfsol'
+crater.GeomInput.solidinput.InputType = "SolidFile"
+crater.GeomInput.solidinput.GeomNames = "domain"
+crater.GeomInput.solidinput.FileName = "crater2D.pfsol"
 
-crater.GeomInput.zone1.InputType = 'Box'
-crater.GeomInput.zone1.GeomName = 'zone1'
+crater.GeomInput.zone1.InputType = "Box"
+crater.GeomInput.zone1.GeomName = "zone1"
 
 crater.Geom.zone1.Lower.X = 0.0
 crater.Geom.zone1.Lower.Y = 0.0
@@ -74,8 +77,8 @@
 crater.Geom.zone1.Upper.Y = 1.0
 crater.Geom.zone1.Upper.Z = 200.0
 
-crater.GeomInput.zone2.InputType = 'Box'
-crater.GeomInput.zone2.GeomName = 'zone2'
+crater.GeomInput.zone2.InputType = "Box"
+crater.GeomInput.zone2.GeomName = "zone2"
 
 crater.Geom.zone2.Lower.X = 0.0
 crater.Geom.zone2.Lower.Y = 0.0
@@ -84,8 +87,8 @@
 crater.Geom.zone2.Upper.Y = 1.0
 crater.Geom.zone2.Upper.Z = 80.0
 
-crater.GeomInput.zone3above4.InputType = 'Box'
-crater.GeomInput.zone3above4.GeomName = 'zone3above4'
+crater.GeomInput.zone3above4.InputType = "Box"
+crater.GeomInput.zone3above4.GeomName = "zone3above4"
 
 crater.Geom.zone3above4.Lower.X = 0.0
 crater.Geom.zone3above4.Lower.Y = 0.0
@@ -94,8 +97,8 @@
 crater.Geom.zone3above4.Upper.Y = 1.0
 crater.Geom.zone3above4.Upper.Z = 200.0
 
-crater.GeomInput.zone3left4.InputType = 'Box'
-crater.GeomInput.zone3left4.GeomName = 'zone3left4'
+crater.GeomInput.zone3left4.InputType = "Box"
+crater.GeomInput.zone3left4.GeomName = "zone3left4"
 
 crater.Geom.zone3left4.Lower.X = 0.0
 crater.Geom.zone3left4.Lower.Y = 0.0
@@ -104,8 +107,8 @@
 crater.Geom.zone3left4.Upper.Y = 1.0
 crater.Geom.zone3left4.Upper.Z = 200.0
 
-crater.GeomInput.zone3right4.InputType = 'Box'
-crater.GeomInput.zone3right4.GeomName = 'zone3right4'
+crater.GeomInput.zone3right4.InputType = "Box"
+crater.GeomInput.zone3right4.GeomName = "zone3right4"
 
 crater.Geom.zone3right4.Lower.X = 30.0
 crater.Geom.zone3right4.Lower.Y = 0.0
@@ -114,8 +117,8 @@
 crater.Geom.zone3right4.Upper.Y = 1.0
 crater.Geom.zone3right4.Upper.Z = 100.0
 
-crater.GeomInput.zone3below4.InputType = 'Box'
-crater.GeomInput.zone3below4.GeomName = 'zone3below4'
+crater.GeomInput.zone3below4.InputType = "Box"
+crater.GeomInput.zone3below4.GeomName = "zone3below4"
 
 crater.Geom.zone3below4.Lower.X = 0.0
 crater.Geom.zone3below4.Lower.Y = 0.0
@@ -124,8 +127,8 @@
 crater.Geom.zone3below4.Upper.Y = 1.0
 crater.Geom.zone3below4.Upper.Z = 20.0
 
-crater.GeomInput.zone4.InputType = 'Box'
-crater.GeomInput.zone4.GeomName = 'zone4'
+crater.GeomInput.zone4.InputType = "Box"
+crater.GeomInput.zone4.GeomName = "zone4"
 
 crater.Geom.zone4.Lower.X = 0.0
 crater.Geom.zone4.Lower.Y = 0.0
@@ -134,8 +137,8 @@
 crater.Geom.zone4.Upper.Y = 1.0
 crater.Geom.zone4.Upper.Z = 150.0
 
-crater.GeomInput.background.InputType = 'Box'
-crater.GeomInput.background.GeomName = 'background'
+crater.GeomInput.background.InputType = "Box"
+crater.GeomInput.background.GeomName = "background"
 
 crater.Geom.background.Lower.X = -99999999.0
 crater.Geom.background.Lower.Y = -99999999.0
@@ -144,131 +147,133 @@
 crater.Geom.background.Upper.Y = 99999999.0
 crater.Geom.background.Upper.Z = 99999999.0
 
-crater.Geom.domain.Patches = 'infiltration z_upper x_lower y_lower x_upper y_upper z_lower'
+crater.Geom.domain.Patches = (
+    "infiltration z_upper x_lower y_lower x_upper y_upper z_lower"
+)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 crater.Geom.Perm.Names = Zones
 
-crater.Geom.zone1.Perm.Type = 'Constant'
+crater.Geom.zone1.Perm.Type = "Constant"
 crater.Geom.zone1.Perm.Value = 9.1496
 
-crater.Geom.zone2.Perm.Type = 'Constant'
+crater.Geom.zone2.Perm.Type = "Constant"
 crater.Geom.zone2.Perm.Value = 5.4427
 
-crater.Geom.zone3above4.Perm.Type = 'Constant'
+crater.Geom.zone3above4.Perm.Type = "Constant"
 crater.Geom.zone3above4.Perm.Value = 4.8033
 
-crater.Geom.zone3left4.Perm.Type = 'Constant'
+crater.Geom.zone3left4.Perm.Type = "Constant"
 crater.Geom.zone3left4.Perm.Value = 4.8033
 
-crater.Geom.zone3right4.Perm.Type = 'Constant'
+crater.Geom.zone3right4.Perm.Type = "Constant"
 crater.Geom.zone3right4.Perm.Value = 4.8033
 
-crater.Geom.zone3below4.Perm.Type = 'Constant'
+crater.Geom.zone3below4.Perm.Type = "Constant"
 crater.Geom.zone3below4.Perm.Value = 4.8033
 
-crater.Geom.zone4.Perm.Type = 'Constant'
-crater.Geom.zone4.Perm.Value = .48033
+crater.Geom.zone4.Perm.Type = "Constant"
+crater.Geom.zone4.Perm.Value = 0.48033
 
-crater.Perm.TensorType = 'TensorByGeom'
+crater.Perm.TensorType = "TensorByGeom"
 
-crater.Geom.Perm.TensorByGeom.Names = 'background'
+crater.Geom.Perm.TensorByGeom.Names = "background"
 
 crater.Geom.background.Perm.TensorValX = 1.0
 crater.Geom.background.Perm.TensorValY = 1.0
 crater.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.SpecificStorage.Type = 'Constant'
-crater.SpecificStorage.GeomNames = 'domain'
+crater.SpecificStorage.Type = "Constant"
+crater.SpecificStorage.GeomNames = "domain"
 crater.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Phase.Names = 'water'
+crater.Phase.Names = "water"
 
-crater.Phase.water.Density.Type = 'Constant'
+crater.Phase.water.Density.Type = "Constant"
 crater.Phase.water.Density.Value = 1.0
 
-crater.Phase.water.Viscosity.Type = 'Constant'
+crater.Phase.water.Viscosity.Type = "Constant"
 crater.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Contaminants.Names = ''
+crater.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Geom.Retardation.GeomNames = ''
+crater.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 crater.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 crater.TimingInfo.BaseUnit = 1.0
 crater.TimingInfo.StartCount = 0
 crater.TimingInfo.StartTime = 0.0
 crater.TimingInfo.StopTime = 300.0
 crater.TimingInfo.DumpInterval = 30.0
-crater.TimeStep.Type = 'Constant'
+crater.TimeStep.Type = "Constant"
 crater.TimeStep.Value = 10.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 crater.Geom.Porosity.GeomNames = Zones
 
-crater.Geom.zone1.Porosity.Type = 'Constant'
+crater.Geom.zone1.Porosity.Type = "Constant"
 crater.Geom.zone1.Porosity.Value = 0.3680
 
-crater.Geom.zone2.Porosity.Type = 'Constant'
+crater.Geom.zone2.Porosity.Type = "Constant"
 crater.Geom.zone2.Porosity.Value = 0.3510
 
-crater.Geom.zone3above4.Porosity.Type = 'Constant'
+crater.Geom.zone3above4.Porosity.Type = "Constant"
 crater.Geom.zone3above4.Porosity.Value = 0.3250
 
-crater.Geom.zone3left4.Porosity.Type = 'Constant'
+crater.Geom.zone3left4.Porosity.Type = "Constant"
 crater.Geom.zone3left4.Porosity.Value = 0.3250
 
-crater.Geom.zone3right4.Porosity.Type = 'Constant'
+crater.Geom.zone3right4.Porosity.Type = "Constant"
 crater.Geom.zone3right4.Porosity.Value = 0.3250
 
-crater.Geom.zone3below4.Porosity.Type = 'Constant'
+crater.Geom.zone3below4.Porosity.Type = "Constant"
 crater.Geom.zone3below4.Porosity.Value = 0.3250
 
-crater.Geom.zone4.Porosity.Type = 'Constant'
+crater.Geom.zone4.Porosity.Type = "Constant"
 crater.Geom.zone4.Porosity.Value = 0.3250
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Domain.GeomName = 'domain'
+crater.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Phase.RelPerm.Type = 'VanGenuchten'
+crater.Phase.RelPerm.Type = "VanGenuchten"
 crater.Phase.RelPerm.GeomNames = Zones
 
 crater.Geom.zone1.RelPerm.Alpha = 3.34
@@ -292,11 +297,11 @@
 crater.Geom.zone4.RelPerm.Alpha = 3.45
 crater.Geom.zone4.RelPerm.N = 1.573
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-crater.Phase.Saturation.Type = 'VanGenuchten'
+crater.Phase.Saturation.Type = "VanGenuchten"
 crater.Phase.Saturation.GeomNames = Zones
 
 crater.Geom.zone1.Saturation.Alpha = 3.34
@@ -339,119 +344,119 @@
 crater.Geom.zone4.Saturation.SRes = 0.2643
 crater.Geom.zone4.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Wells.Names = ''
+crater.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Cycle.Names = 'constant onoff'
-crater.Cycle.constant.Names = 'alltime'
+crater.Cycle.Names = "constant onoff"
+crater.Cycle.constant.Names = "alltime"
 crater.Cycle.constant.alltime.Length = 1
 crater.Cycle.constant.Repeat = -1
 
-crater.Cycle.onoff.Names = 'on off'
+crater.Cycle.onoff.Names = "on off"
 crater.Cycle.onoff.on.Length = 10
 crater.Cycle.onoff.off.Length = 90
 crater.Cycle.onoff.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 crater.BCPressure.PatchNames = crater.Geom.domain.Patches
 
-crater.Patch.infiltration.BCPressure.Type = 'FluxConst'
-crater.Patch.infiltration.BCPressure.Cycle = 'onoff'
+crater.Patch.infiltration.BCPressure.Type = "FluxConst"
+crater.Patch.infiltration.BCPressure.Cycle = "onoff"
 crater.Patch.infiltration.BCPressure.on.Value = -0.10
 crater.Patch.infiltration.BCPressure.off.Value = 0.0
 
-crater.Patch.x_lower.BCPressure.Type = 'FluxConst'
-crater.Patch.x_lower.BCPressure.Cycle = 'constant'
+crater.Patch.x_lower.BCPressure.Type = "FluxConst"
+crater.Patch.x_lower.BCPressure.Cycle = "constant"
 crater.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-crater.Patch.y_lower.BCPressure.Type = 'FluxConst'
-crater.Patch.y_lower.BCPressure.Cycle = 'constant'
+crater.Patch.y_lower.BCPressure.Type = "FluxConst"
+crater.Patch.y_lower.BCPressure.Cycle = "constant"
 crater.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-crater.Patch.z_lower.BCPressure.Type = 'FluxConst'
-crater.Patch.z_lower.BCPressure.Cycle = 'constant'
+crater.Patch.z_lower.BCPressure.Type = "FluxConst"
+crater.Patch.z_lower.BCPressure.Cycle = "constant"
 crater.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-crater.Patch.x_upper.BCPressure.Type = 'FluxConst'
-crater.Patch.x_upper.BCPressure.Cycle = 'constant'
+crater.Patch.x_upper.BCPressure.Type = "FluxConst"
+crater.Patch.x_upper.BCPressure.Cycle = "constant"
 crater.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-crater.Patch.y_upper.BCPressure.Type = 'FluxConst'
-crater.Patch.y_upper.BCPressure.Cycle = 'constant'
+crater.Patch.y_upper.BCPressure.Type = "FluxConst"
+crater.Patch.y_upper.BCPressure.Cycle = "constant"
 crater.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-crater.Patch.z_upper.BCPressure.Type = 'FluxConst'
-crater.Patch.z_upper.BCPressure.Cycle = 'constant'
+crater.Patch.z_upper.BCPressure.Type = "FluxConst"
+crater.Patch.z_upper.BCPressure.Cycle = "constant"
 crater.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-crater.TopoSlopesX.Type = 'Constant'
-crater.TopoSlopesX.GeomNames = 'domain'
+crater.TopoSlopesX.Type = "Constant"
+crater.TopoSlopesX.GeomNames = "domain"
 crater.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-crater.TopoSlopesY.Type = 'Constant'
-crater.TopoSlopesY.GeomNames = 'domain'
+crater.TopoSlopesY.Type = "Constant"
+crater.TopoSlopesY.GeomNames = "domain"
 crater.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-crater.Mannings.Type = 'Constant'
-crater.Mannings.GeomNames = 'domain'
-crater.Mannings.Geom.domain.Value = 0.
+crater.Mannings.Type = "Constant"
+crater.Mannings.GeomNames = "domain"
+crater.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-crater.ICPressure.Type = 'HydroStaticPatch'
-crater.ICPressure.GeomNames = 'domain'
+crater.ICPressure.Type = "HydroStaticPatch"
+crater.ICPressure.GeomNames = "domain"
 
 crater.Geom.domain.ICPressure.Value = 1.0
-crater.Geom.domain.ICPressure.RefPatch = 'z_lower'
-crater.Geom.domain.ICPressure.RefGeom = 'domain'
+crater.Geom.domain.ICPressure.RefPatch = "z_lower"
+crater.Geom.domain.ICPressure.RefGeom = "domain"
 
 crater.Geom.infiltration.ICPressure.Value = 10.0
-crater.Geom.infiltration.ICPressure.RefPatch = 'infiltration'
-crater.Geom.infiltration.ICPressure.RefGeom = 'domain'
+crater.Geom.infiltration.ICPressure.RefPatch = "infiltration"
+crater.Geom.infiltration.ICPressure.RefGeom = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.PhaseSources.water.Type = 'Constant'
-crater.PhaseSources.water.GeomNames = 'background'
+crater.PhaseSources.water.Type = "Constant"
+crater.PhaseSources.water.GeomNames = "background"
 crater.PhaseSources.water.Geom.background.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.KnownSolution = 'NoKnownSolution'
+crater.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Solver = 'Richards'
+crater.Solver = "Richards"
 crater.Solver.MaxIter = 10000
 
 crater.Solver.Nonlinear.MaxIter = 15
@@ -464,12 +469,52 @@
 crater.Solver.Linear.KrylovDimension = 25
 crater.Solver.Linear.MaxRestarts = 2
 
-crater.Solver.Linear.Preconditioner = 'MGSemi'
+crater.Solver.Linear.Preconditioner = "MGSemi"
 crater.Solver.Linear.Preconditioner.MGSemi.MaxIter = 1
 crater.Solver.Linear.Preconditioner.MGSemi.MaxLevels = 100
 
-#-----------------------------------------------------------------------------
-# Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
+crater.Solver.PrintTop = True
 
-crater.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+# Run and Unload the ParFlow output files
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = dir_name
+correct_output_dir_name = get_absolute_path("../correct_output")
+crater.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z", "porosity", "top_patch", "top_zindex"]
+
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+for i in range(11):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/crater2D_vangtable_linear.py b/test/python/crater2D_vangtable_linear.py
index 3d652abdb..eee0933e7 100644
--- a/test/python/crater2D_vangtable_linear.py
+++ b/test/python/crater2D_vangtable_linear.py
@@ -1,24 +1,27 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 #  This is a 2D crater problem w/ time varying input and topography
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import cp, mkdir, chdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
-crater = Run("crater2D_vangtable_linear", __file__)
+run_name = "crater2D_vangtable_linear"
+crater = Run(run_name, __file__)
 
 # ---------------------------------------------------------
 # Copy testing data in test directory
 # ---------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/crater_vl')
+dir_name = get_absolute_path("test_output/crater_vl")
 mkdir(dir_name)
 
-cp('$PF_SRC/test/input/crater2D.pfsol', dir_name)
+cp("$PF_SRC/test/input/crater2D.pfsol", dir_name)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Controls for the Van Genuchten curves used later.
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Note this problem was setup to use an easy curve for the linear
 # interpolation. This way the test will match closely between direct
 # evaluation and when using linear interpolation avoiding the
@@ -31,7 +34,7 @@
 VG_N = 2.0
 VG_interpolation_method = "Linear"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 crater.FileVersion = 4
 
@@ -39,9 +42,9 @@
 crater.Process.Topology.Q = 1
 crater.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 crater.ComputationalGrid.Lower.X = 0.0
 crater.ComputationalGrid.Lower.Y = 0.0
@@ -67,20 +70,20 @@
 crater.ComputationalGrid.DY = (UpperY - LowerY) / NY
 crater.ComputationalGrid.DZ = (UpperZ - LowerZ) / NZ
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 Zones = "zone1 zone2 zone3above4 zone3left4 zone3right4 zone3below4 zone4"
 
-crater.GeomInput.Names = f'solidinput {Zones} background'
+crater.GeomInput.Names = f"solidinput {Zones} background"
 
-crater.GeomInput.solidinput.InputType = 'SolidFile'
-crater.GeomInput.solidinput.GeomNames = 'domain'
-crater.GeomInput.solidinput.FileName = 'crater2D.pfsol'
+crater.GeomInput.solidinput.InputType = "SolidFile"
+crater.GeomInput.solidinput.GeomNames = "domain"
+crater.GeomInput.solidinput.FileName = "crater2D.pfsol"
 
-crater.GeomInput.zone1.InputType = 'Box'
-crater.GeomInput.zone1.GeomName = 'zone1'
+crater.GeomInput.zone1.InputType = "Box"
+crater.GeomInput.zone1.GeomName = "zone1"
 
 crater.Geom.zone1.Lower.X = 0.0
 crater.Geom.zone1.Lower.Y = 0.0
@@ -89,8 +92,8 @@
 crater.Geom.zone1.Upper.Y = 1.0
 crater.Geom.zone1.Upper.Z = 200.0
 
-crater.GeomInput.zone2.InputType = 'Box'
-crater.GeomInput.zone2.GeomName = 'zone2'
+crater.GeomInput.zone2.InputType = "Box"
+crater.GeomInput.zone2.GeomName = "zone2"
 
 crater.Geom.zone2.Lower.X = 0.0
 crater.Geom.zone2.Lower.Y = 0.0
@@ -99,8 +102,8 @@
 crater.Geom.zone2.Upper.Y = 1.0
 crater.Geom.zone2.Upper.Z = 80.0
 
-crater.GeomInput.zone3above4.InputType = 'Box'
-crater.GeomInput.zone3above4.GeomName = 'zone3above4'
+crater.GeomInput.zone3above4.InputType = "Box"
+crater.GeomInput.zone3above4.GeomName = "zone3above4"
 
 crater.Geom.zone3above4.Lower.X = 0.0
 crater.Geom.zone3above4.Lower.Y = 0.0
@@ -109,8 +112,8 @@
 crater.Geom.zone3above4.Upper.Y = 1.0
 crater.Geom.zone3above4.Upper.Z = 200.0
 
-crater.GeomInput.zone3left4.InputType = 'Box'
-crater.GeomInput.zone3left4.GeomName = 'zone3left4'
+crater.GeomInput.zone3left4.InputType = "Box"
+crater.GeomInput.zone3left4.GeomName = "zone3left4"
 
 crater.Geom.zone3left4.Lower.X = 0.0
 crater.Geom.zone3left4.Lower.Y = 0.0
@@ -119,8 +122,8 @@
 crater.Geom.zone3left4.Upper.Y = 1.0
 crater.Geom.zone3left4.Upper.Z = 200.0
 
-crater.GeomInput.zone3right4.InputType = 'Box'
-crater.GeomInput.zone3right4.GeomName = 'zone3right4'
+crater.GeomInput.zone3right4.InputType = "Box"
+crater.GeomInput.zone3right4.GeomName = "zone3right4"
 
 crater.Geom.zone3right4.Lower.X = 30.0
 crater.Geom.zone3right4.Lower.Y = 0.0
@@ -129,8 +132,8 @@
 crater.Geom.zone3right4.Upper.Y = 1.0
 crater.Geom.zone3right4.Upper.Z = 100.0
 
-crater.GeomInput.zone3below4.InputType = 'Box'
-crater.GeomInput.zone3below4.GeomName = 'zone3below4'
+crater.GeomInput.zone3below4.InputType = "Box"
+crater.GeomInput.zone3below4.GeomName = "zone3below4"
 
 crater.Geom.zone3below4.Lower.X = 0.0
 crater.Geom.zone3below4.Lower.Y = 0.0
@@ -139,8 +142,8 @@
 crater.Geom.zone3below4.Upper.Y = 1.0
 crater.Geom.zone3below4.Upper.Z = 20.0
 
-crater.GeomInput.zone4.InputType = 'Box'
-crater.GeomInput.zone4.GeomName = 'zone4'
+crater.GeomInput.zone4.InputType = "Box"
+crater.GeomInput.zone4.GeomName = "zone4"
 
 crater.Geom.zone4.Lower.X = 0.0
 crater.Geom.zone4.Lower.Y = 0.0
@@ -149,8 +152,8 @@
 crater.Geom.zone4.Upper.Y = 1.0
 crater.Geom.zone4.Upper.Z = 150.0
 
-crater.GeomInput.background.InputType = 'Box'
-crater.GeomInput.background.GeomName = 'background'
+crater.GeomInput.background.InputType = "Box"
+crater.GeomInput.background.GeomName = "background"
 
 crater.Geom.background.Lower.X = -99999999.0
 crater.Geom.background.Lower.Y = -99999999.0
@@ -159,132 +162,132 @@
 crater.Geom.background.Upper.Y = 99999999.0
 crater.Geom.background.Upper.Z = 99999999.0
 
-crater.Geom.domain.Patches = 'infiltration z_upper x_lower y_lower \
-                                       x_upper y_upper z_lower'
+crater.Geom.domain.Patches = "infiltration z_upper x_lower y_lower \
+                                       x_upper y_upper z_lower"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 crater.Geom.Perm.Names = Zones
 
-crater.Geom.zone1.Perm.Type = 'Constant'
+crater.Geom.zone1.Perm.Type = "Constant"
 crater.Geom.zone1.Perm.Value = 9.1496
 
-crater.Geom.zone2.Perm.Type = 'Constant'
+crater.Geom.zone2.Perm.Type = "Constant"
 crater.Geom.zone2.Perm.Value = 5.4427
 
-crater.Geom.zone3above4.Perm.Type = 'Constant'
+crater.Geom.zone3above4.Perm.Type = "Constant"
 crater.Geom.zone3above4.Perm.Value = 4.8033
 
-crater.Geom.zone3left4.Perm.Type = 'Constant'
+crater.Geom.zone3left4.Perm.Type = "Constant"
 crater.Geom.zone3left4.Perm.Value = 4.8033
 
-crater.Geom.zone3right4.Perm.Type = 'Constant'
+crater.Geom.zone3right4.Perm.Type = "Constant"
 crater.Geom.zone3right4.Perm.Value = 4.8033
 
-crater.Geom.zone3below4.Perm.Type = 'Constant'
+crater.Geom.zone3below4.Perm.Type = "Constant"
 crater.Geom.zone3below4.Perm.Value = 4.8033
 
-crater.Geom.zone4.Perm.Type = 'Constant'
-crater.Geom.zone4.Perm.Value = .48033
+crater.Geom.zone4.Perm.Type = "Constant"
+crater.Geom.zone4.Perm.Value = 0.48033
 
-crater.Perm.TensorType = 'TensorByGeom'
+crater.Perm.TensorType = "TensorByGeom"
 
-crater.Geom.Perm.TensorByGeom.Names = 'background'
+crater.Geom.Perm.TensorByGeom.Names = "background"
 
 crater.Geom.background.Perm.TensorValX = 1.0
 crater.Geom.background.Perm.TensorValY = 1.0
 crater.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.SpecificStorage.Type = 'Constant'
-crater.SpecificStorage.GeomNames = 'domain'
+crater.SpecificStorage.Type = "Constant"
+crater.SpecificStorage.GeomNames = "domain"
 crater.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Phase.Names = 'water'
+crater.Phase.Names = "water"
 
-crater.Phase.water.Density.Type = 'Constant'
+crater.Phase.water.Density.Type = "Constant"
 crater.Phase.water.Density.Value = 1.0
 
-crater.Phase.water.Viscosity.Type = 'Constant'
+crater.Phase.water.Viscosity.Type = "Constant"
 crater.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Contaminants.Names = ''
+crater.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Geom.Retardation.GeomNames = ''
+crater.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 crater.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 crater.TimingInfo.BaseUnit = 1.0
 crater.TimingInfo.StartCount = 0
 crater.TimingInfo.StartTime = 0.0
 crater.TimingInfo.StopTime = 20.0
 crater.TimingInfo.DumpInterval = 10.0
-crater.TimeStep.Type = 'Constant'
+crater.TimeStep.Type = "Constant"
 crater.TimeStep.Value = 10.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 crater.Geom.Porosity.GeomNames = Zones
 
-crater.Geom.zone1.Porosity.Type = 'Constant'
+crater.Geom.zone1.Porosity.Type = "Constant"
 crater.Geom.zone1.Porosity.Value = 0.3680
 
-crater.Geom.zone2.Porosity.Type = 'Constant'
+crater.Geom.zone2.Porosity.Type = "Constant"
 crater.Geom.zone2.Porosity.Value = 0.3510
 
-crater.Geom.zone3above4.Porosity.Type = 'Constant'
+crater.Geom.zone3above4.Porosity.Type = "Constant"
 crater.Geom.zone3above4.Porosity.Value = 0.3250
 
-crater.Geom.zone3left4.Porosity.Type = 'Constant'
+crater.Geom.zone3left4.Porosity.Type = "Constant"
 crater.Geom.zone3left4.Porosity.Value = 0.3250
 
-crater.Geom.zone3right4.Porosity.Type = 'Constant'
+crater.Geom.zone3right4.Porosity.Type = "Constant"
 crater.Geom.zone3right4.Porosity.Value = 0.3250
 
-crater.Geom.zone3below4.Porosity.Type = 'Constant'
+crater.Geom.zone3below4.Porosity.Type = "Constant"
 crater.Geom.zone3below4.Porosity.Value = 0.3250
 
-crater.Geom.zone4.Porosity.Type = 'Constant'
+crater.Geom.zone4.Porosity.Type = "Constant"
 crater.Geom.zone4.Porosity.Value = 0.3250
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Domain.GeomName = 'domain'
+crater.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Phase.RelPerm.Type = 'VanGenuchten'
+crater.Phase.RelPerm.Type = "VanGenuchten"
 crater.Phase.RelPerm.GeomNames = Zones
 
 crater.Geom.zone1.RelPerm.Alpha = VG_alpha
@@ -329,11 +332,11 @@
 crater.Geom.zone4.RelPerm.MinPressureHead = -300
 crater.Geom.zone4.RelPerm.InterpolationMethod = VG_interpolation_method
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-crater.Phase.Saturation.Type = 'VanGenuchten'
+crater.Phase.Saturation.Type = "VanGenuchten"
 crater.Phase.Saturation.GeomNames = Zones
 
 crater.Geom.zone1.Saturation.Alpha = VG_alpha
@@ -371,121 +374,121 @@
 crater.Geom.zone4.Saturation.SRes = 0.2643
 crater.Geom.zone4.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Wells.Names = ''
+crater.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Cycle.Names = 'constant onoff'
-crater.Cycle.constant.Names = 'alltime'
+crater.Cycle.Names = "constant onoff"
+crater.Cycle.constant.Names = "alltime"
 crater.Cycle.constant.alltime.Length = 1
 crater.Cycle.constant.Repeat = -1
 
-crater.Cycle.onoff.Names = 'on off'
+crater.Cycle.onoff.Names = "on off"
 crater.Cycle.onoff.on.Length = 10
 crater.Cycle.onoff.off.Length = 90
 crater.Cycle.onoff.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 crater.BCPressure.PatchNames = crater.Geom.domain.Patches
 
-crater.Patch.infiltration.BCPressure.Type = 'FluxConst'
-crater.Patch.infiltration.BCPressure.Cycle = 'onoff'
+crater.Patch.infiltration.BCPressure.Type = "FluxConst"
+crater.Patch.infiltration.BCPressure.Cycle = "onoff"
 crater.Patch.infiltration.BCPressure.on.Value = -0.10
 crater.Patch.infiltration.BCPressure.off.Value = 0.0
 
-crater.Patch.x_lower.BCPressure.Type = 'FluxConst'
-crater.Patch.x_lower.BCPressure.Cycle = 'constant'
+crater.Patch.x_lower.BCPressure.Type = "FluxConst"
+crater.Patch.x_lower.BCPressure.Cycle = "constant"
 crater.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-crater.Patch.y_lower.BCPressure.Type = 'FluxConst'
-crater.Patch.y_lower.BCPressure.Cycle = 'constant'
+crater.Patch.y_lower.BCPressure.Type = "FluxConst"
+crater.Patch.y_lower.BCPressure.Cycle = "constant"
 crater.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-crater.Patch.z_lower.BCPressure.Type = 'FluxConst'
-crater.Patch.z_lower.BCPressure.Cycle = 'constant'
+crater.Patch.z_lower.BCPressure.Type = "FluxConst"
+crater.Patch.z_lower.BCPressure.Cycle = "constant"
 crater.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-crater.Patch.x_upper.BCPressure.Type = 'FluxConst'
-crater.Patch.x_upper.BCPressure.Cycle = 'constant'
+crater.Patch.x_upper.BCPressure.Type = "FluxConst"
+crater.Patch.x_upper.BCPressure.Cycle = "constant"
 crater.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-crater.Patch.y_upper.BCPressure.Type = 'FluxConst'
-crater.Patch.y_upper.BCPressure.Cycle = 'constant'
+crater.Patch.y_upper.BCPressure.Type = "FluxConst"
+crater.Patch.y_upper.BCPressure.Cycle = "constant"
 crater.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-crater.Patch.z_upper.BCPressure.Type = 'FluxConst'
-crater.Patch.z_upper.BCPressure.Cycle = 'constant'
+crater.Patch.z_upper.BCPressure.Type = "FluxConst"
+crater.Patch.z_upper.BCPressure.Cycle = "constant"
 crater.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-crater.TopoSlopesX.Type = 'Constant'
-crater.TopoSlopesX.GeomNames = 'domain'
+crater.TopoSlopesX.Type = "Constant"
+crater.TopoSlopesX.GeomNames = "domain"
 
 crater.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-crater.TopoSlopesY.Type = 'Constant'
-crater.TopoSlopesY.GeomNames = 'domain'
+crater.TopoSlopesY.Type = "Constant"
+crater.TopoSlopesY.GeomNames = "domain"
 
 crater.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-crater.Mannings.Type = 'Constant'
-crater.Mannings.GeomNames = 'domain'
-crater.Mannings.Geom.domain.Value = 0.
+crater.Mannings.Type = "Constant"
+crater.Mannings.GeomNames = "domain"
+crater.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-crater.ICPressure.Type = 'HydroStaticPatch'
-crater.ICPressure.GeomNames = 'domain'
+crater.ICPressure.Type = "HydroStaticPatch"
+crater.ICPressure.GeomNames = "domain"
 
 crater.Geom.domain.ICPressure.Value = 1.0
-crater.Geom.domain.ICPressure.RefPatch = 'z_lower'
-crater.Geom.domain.ICPressure.RefGeom = 'domain'
+crater.Geom.domain.ICPressure.RefPatch = "z_lower"
+crater.Geom.domain.ICPressure.RefGeom = "domain"
 
 crater.Geom.infiltration.ICPressure.Value = 10.0
-crater.Geom.infiltration.ICPressure.RefPatch = 'infiltration'
-crater.Geom.infiltration.ICPressure.RefGeom = 'domain'
+crater.Geom.infiltration.ICPressure.RefPatch = "infiltration"
+crater.Geom.infiltration.ICPressure.RefGeom = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.PhaseSources.water.Type = 'Constant'
-crater.PhaseSources.water.GeomNames = 'background'
+crater.PhaseSources.water.Type = "Constant"
+crater.PhaseSources.water.GeomNames = "background"
 crater.PhaseSources.water.Geom.background.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.KnownSolution = 'NoKnownSolution'
+crater.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Solver = 'Richards'
+crater.Solver = "Richards"
 crater.Solver.MaxIter = 10000
 
 crater.Solver.Nonlinear.MaxIter = 15
@@ -498,12 +501,52 @@
 crater.Solver.Linear.KrylovDimension = 25
 crater.Solver.Linear.MaxRestarts = 10
 
-crater.Solver.Linear.Preconditioner = 'MGSemi'
+crater.Solver.Linear.Preconditioner = "MGSemi"
 crater.Solver.Linear.Preconditioner.MGSemi.MaxIter = 1
 crater.Solver.Linear.Preconditioner.MGSemi.MaxLevels = 100
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-crater.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = dir_name
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+crater.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z", "alpha", "n", "sres", "ssat"]
+
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+for i in range(3):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/crater2D_vangtable_spline.py b/test/python/crater2D_vangtable_spline.py
index b6005f00e..dde0a556d 100644
--- a/test/python/crater2D_vangtable_spline.py
+++ b/test/python/crater2D_vangtable_spline.py
@@ -1,30 +1,33 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 #  This is a 2D crater problem w/ time varying input and topography
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import cp, mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file, pf_test_file_with_abs
 
-crater = Run("crater2D_vangtable_spline", __file__)
+run_name = "crater2D_vangtable_spline"
+crater = Run(run_name, __file__)
 
 # ---------------------------------------------------------
 # Copy testing data in test directory
 # ---------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/crater_vs')
+dir_name = get_absolute_path("test_output/crater_vs")
 mkdir(dir_name)
 
-cp('$PF_SRC/test/input/crater2D.pfsol', dir_name)
+cp("$PF_SRC/test/input/crater2D.pfsol", dir_name)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Controls for the VanG curves used later.
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 VG_points = 20000
 VG_alpha = 1.0
 VG_N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 crater.FileVersion = 4
 
@@ -32,9 +35,9 @@
 crater.Process.Topology.Q = 1
 crater.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 crater.ComputationalGrid.Lower.X = 0.0
 crater.ComputationalGrid.Lower.Y = 0.0
@@ -60,20 +63,20 @@
 crater.ComputationalGrid.DY = (UpperY - LowerY) / NY
 crater.ComputationalGrid.DZ = (UpperZ - LowerZ) / NZ
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 Zones = "zone1 zone2 zone3above4 zone3left4 zone3right4 zone3below4 zone4"
 
-crater.GeomInput.Names = f'solidinput {Zones} background'
+crater.GeomInput.Names = f"solidinput {Zones} background"
 
-crater.GeomInput.solidinput.InputType = 'SolidFile'
-crater.GeomInput.solidinput.GeomNames = 'domain'
-crater.GeomInput.solidinput.FileName = 'crater2D.pfsol'
+crater.GeomInput.solidinput.InputType = "SolidFile"
+crater.GeomInput.solidinput.GeomNames = "domain"
+crater.GeomInput.solidinput.FileName = "crater2D.pfsol"
 
-crater.GeomInput.zone1.InputType = 'Box'
-crater.GeomInput.zone1.GeomName = 'zone1'
+crater.GeomInput.zone1.InputType = "Box"
+crater.GeomInput.zone1.GeomName = "zone1"
 
 crater.Geom.zone1.Lower.X = 0.0
 crater.Geom.zone1.Lower.Y = 0.0
@@ -82,8 +85,8 @@
 crater.Geom.zone1.Upper.Y = 1.0
 crater.Geom.zone1.Upper.Z = 200.0
 
-crater.GeomInput.zone2.InputType = 'Box'
-crater.GeomInput.zone2.GeomName = 'zone2'
+crater.GeomInput.zone2.InputType = "Box"
+crater.GeomInput.zone2.GeomName = "zone2"
 
 crater.Geom.zone2.Lower.X = 0.0
 crater.Geom.zone2.Lower.Y = 0.0
@@ -92,8 +95,8 @@
 crater.Geom.zone2.Upper.Y = 1.0
 crater.Geom.zone2.Upper.Z = 80.0
 
-crater.GeomInput.zone3above4.InputType = 'Box'
-crater.GeomInput.zone3above4.GeomName = 'zone3above4'
+crater.GeomInput.zone3above4.InputType = "Box"
+crater.GeomInput.zone3above4.GeomName = "zone3above4"
 
 crater.Geom.zone3above4.Lower.X = 0.0
 crater.Geom.zone3above4.Lower.Y = 0.0
@@ -102,8 +105,8 @@
 crater.Geom.zone3above4.Upper.Y = 1.0
 crater.Geom.zone3above4.Upper.Z = 200.0
 
-crater.GeomInput.zone3left4.InputType = 'Box'
-crater.GeomInput.zone3left4.GeomName = 'zone3left4'
+crater.GeomInput.zone3left4.InputType = "Box"
+crater.GeomInput.zone3left4.GeomName = "zone3left4"
 
 crater.Geom.zone3left4.Lower.X = 0.0
 crater.Geom.zone3left4.Lower.Y = 0.0
@@ -112,8 +115,8 @@
 crater.Geom.zone3left4.Upper.Y = 1.0
 crater.Geom.zone3left4.Upper.Z = 200.0
 
-crater.GeomInput.zone3right4.InputType = 'Box'
-crater.GeomInput.zone3right4.GeomName = 'zone3right4'
+crater.GeomInput.zone3right4.InputType = "Box"
+crater.GeomInput.zone3right4.GeomName = "zone3right4"
 
 crater.Geom.zone3right4.Lower.X = 30.0
 crater.Geom.zone3right4.Lower.Y = 0.0
@@ -122,8 +125,8 @@
 crater.Geom.zone3right4.Upper.Y = 1.0
 crater.Geom.zone3right4.Upper.Z = 100.0
 
-crater.GeomInput.zone3below4.InputType = 'Box'
-crater.GeomInput.zone3below4.GeomName = 'zone3below4'
+crater.GeomInput.zone3below4.InputType = "Box"
+crater.GeomInput.zone3below4.GeomName = "zone3below4"
 
 crater.Geom.zone3below4.Lower.X = 0.0
 crater.Geom.zone3below4.Lower.Y = 0.0
@@ -132,8 +135,8 @@
 crater.Geom.zone3below4.Upper.Y = 1.0
 crater.Geom.zone3below4.Upper.Z = 20.0
 
-crater.GeomInput.zone4.InputType = 'Box'
-crater.GeomInput.zone4.GeomName = 'zone4'
+crater.GeomInput.zone4.InputType = "Box"
+crater.GeomInput.zone4.GeomName = "zone4"
 
 crater.Geom.zone4.Lower.X = 0.0
 crater.Geom.zone4.Lower.Y = 0.0
@@ -142,8 +145,8 @@
 crater.Geom.zone4.Upper.Y = 1.0
 crater.Geom.zone4.Upper.Z = 150.0
 
-crater.GeomInput.background.InputType = 'Box'
-crater.GeomInput.background.GeomName = 'background'
+crater.GeomInput.background.InputType = "Box"
+crater.GeomInput.background.GeomName = "background"
 
 crater.Geom.background.Lower.X = -99999999.0
 crater.Geom.background.Lower.Y = -99999999.0
@@ -152,132 +155,132 @@
 crater.Geom.background.Upper.Y = 99999999.0
 crater.Geom.background.Upper.Z = 99999999.0
 
-crater.Geom.domain.Patches = 'infiltration z_upper x_lower y_lower \
-    x_upper y_upper z_lower'
+crater.Geom.domain.Patches = "infiltration z_upper x_lower y_lower \
+    x_upper y_upper z_lower"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 crater.Geom.Perm.Names = Zones
 
-crater.Geom.zone1.Perm.Type = 'Constant'
+crater.Geom.zone1.Perm.Type = "Constant"
 crater.Geom.zone1.Perm.Value = 9.1496
 
-crater.Geom.zone2.Perm.Type = 'Constant'
+crater.Geom.zone2.Perm.Type = "Constant"
 crater.Geom.zone2.Perm.Value = 5.4427
 
-crater.Geom.zone3above4.Perm.Type = 'Constant'
+crater.Geom.zone3above4.Perm.Type = "Constant"
 crater.Geom.zone3above4.Perm.Value = 4.8033
 
-crater.Geom.zone3left4.Perm.Type = 'Constant'
+crater.Geom.zone3left4.Perm.Type = "Constant"
 crater.Geom.zone3left4.Perm.Value = 4.8033
 
-crater.Geom.zone3right4.Perm.Type = 'Constant'
+crater.Geom.zone3right4.Perm.Type = "Constant"
 crater.Geom.zone3right4.Perm.Value = 4.8033
 
-crater.Geom.zone3below4.Perm.Type = 'Constant'
+crater.Geom.zone3below4.Perm.Type = "Constant"
 crater.Geom.zone3below4.Perm.Value = 4.8033
 
-crater.Geom.zone4.Perm.Type = 'Constant'
-crater.Geom.zone4.Perm.Value = .48033
+crater.Geom.zone4.Perm.Type = "Constant"
+crater.Geom.zone4.Perm.Value = 0.48033
 
-crater.Perm.TensorType = 'TensorByGeom'
+crater.Perm.TensorType = "TensorByGeom"
 
-crater.Geom.Perm.TensorByGeom.Names = 'background'
+crater.Geom.Perm.TensorByGeom.Names = "background"
 
 crater.Geom.background.Perm.TensorValX = 1.0
 crater.Geom.background.Perm.TensorValY = 1.0
 crater.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.SpecificStorage.Type = 'Constant'
-crater.SpecificStorage.GeomNames = 'domain'
+crater.SpecificStorage.Type = "Constant"
+crater.SpecificStorage.GeomNames = "domain"
 crater.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Phase.Names = 'water'
+crater.Phase.Names = "water"
 
-crater.Phase.water.Density.Type = 'Constant'
+crater.Phase.water.Density.Type = "Constant"
 crater.Phase.water.Density.Value = 1.0
 
-crater.Phase.water.Viscosity.Type = 'Constant'
+crater.Phase.water.Viscosity.Type = "Constant"
 crater.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Contaminants.Names = ''
+crater.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Geom.Retardation.GeomNames = ''
+crater.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 crater.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 crater.TimingInfo.BaseUnit = 1.0
 crater.TimingInfo.StartCount = 0
 crater.TimingInfo.StartTime = 0.0
 crater.TimingInfo.StopTime = 20.0
 crater.TimingInfo.DumpInterval = 10.0
-crater.TimeStep.Type = 'Constant'
+crater.TimeStep.Type = "Constant"
 crater.TimeStep.Value = 10.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 crater.Geom.Porosity.GeomNames = Zones
 
-crater.Geom.zone1.Porosity.Type = 'Constant'
+crater.Geom.zone1.Porosity.Type = "Constant"
 crater.Geom.zone1.Porosity.Value = 0.3680
 
-crater.Geom.zone2.Porosity.Type = 'Constant'
+crater.Geom.zone2.Porosity.Type = "Constant"
 crater.Geom.zone2.Porosity.Value = 0.3510
 
-crater.Geom.zone3above4.Porosity.Type = 'Constant'
+crater.Geom.zone3above4.Porosity.Type = "Constant"
 crater.Geom.zone3above4.Porosity.Value = 0.3250
 
-crater.Geom.zone3left4.Porosity.Type = 'Constant'
+crater.Geom.zone3left4.Porosity.Type = "Constant"
 crater.Geom.zone3left4.Porosity.Value = 0.3250
 
-crater.Geom.zone3right4.Porosity.Type = 'Constant'
+crater.Geom.zone3right4.Porosity.Type = "Constant"
 crater.Geom.zone3right4.Porosity.Value = 0.3250
 
-crater.Geom.zone3below4.Porosity.Type = 'Constant'
+crater.Geom.zone3below4.Porosity.Type = "Constant"
 crater.Geom.zone3below4.Porosity.Value = 0.3250
 
-crater.Geom.zone4.Porosity.Type = 'Constant'
+crater.Geom.zone4.Porosity.Type = "Constant"
 crater.Geom.zone4.Porosity.Value = 0.3250
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Domain.GeomName = 'domain'
+crater.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Phase.RelPerm.Type = 'VanGenuchten'
+crater.Phase.RelPerm.Type = "VanGenuchten"
 crater.Phase.RelPerm.GeomNames = Zones
 
 crater.Geom.zone1.RelPerm.Alpha = VG_alpha
@@ -315,11 +318,11 @@
 crater.Geom.zone4.RelPerm.NumSamplePoints = VG_points
 crater.Geom.zone4.RelPerm.MinPressureHead = -300
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-crater.Phase.Saturation.Type = 'VanGenuchten'
+crater.Phase.Saturation.Type = "VanGenuchten"
 crater.Phase.Saturation.GeomNames = Zones
 
 crater.Geom.zone1.Saturation.Alpha = VG_alpha
@@ -357,118 +360,118 @@
 crater.Geom.zone4.Saturation.SRes = 0.2643
 crater.Geom.zone4.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Wells.Names = ''
+crater.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.Cycle.Names = 'constant onoff'
-crater.Cycle.constant.Names = 'alltime'
+crater.Cycle.Names = "constant onoff"
+crater.Cycle.constant.Names = "alltime"
 crater.Cycle.constant.alltime.Length = 1
 crater.Cycle.constant.Repeat = -1
 
-crater.Cycle.onoff.Names = 'on off'
+crater.Cycle.onoff.Names = "on off"
 crater.Cycle.onoff.on.Length = 10
 crater.Cycle.onoff.off.Length = 90
 crater.Cycle.onoff.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 crater.BCPressure.PatchNames = crater.Geom.domain.Patches
 
-crater.Patch.infiltration.BCPressure.Type = 'FluxConst'
-crater.Patch.infiltration.BCPressure.Cycle = 'onoff'
+crater.Patch.infiltration.BCPressure.Type = "FluxConst"
+crater.Patch.infiltration.BCPressure.Cycle = "onoff"
 crater.Patch.infiltration.BCPressure.on.Value = -0.10
 crater.Patch.infiltration.BCPressure.off.Value = 0.0
 
-crater.Patch.x_lower.BCPressure.Type = 'FluxConst'
-crater.Patch.x_lower.BCPressure.Cycle = 'constant'
+crater.Patch.x_lower.BCPressure.Type = "FluxConst"
+crater.Patch.x_lower.BCPressure.Cycle = "constant"
 crater.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-crater.Patch.y_lower.BCPressure.Type = 'FluxConst'
-crater.Patch.y_lower.BCPressure.Cycle = 'constant'
+crater.Patch.y_lower.BCPressure.Type = "FluxConst"
+crater.Patch.y_lower.BCPressure.Cycle = "constant"
 crater.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-crater.Patch.z_lower.BCPressure.Type = 'FluxConst'
-crater.Patch.z_lower.BCPressure.Cycle = 'constant'
+crater.Patch.z_lower.BCPressure.Type = "FluxConst"
+crater.Patch.z_lower.BCPressure.Cycle = "constant"
 crater.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-crater.Patch.x_upper.BCPressure.Type = 'FluxConst'
-crater.Patch.x_upper.BCPressure.Cycle = 'constant'
+crater.Patch.x_upper.BCPressure.Type = "FluxConst"
+crater.Patch.x_upper.BCPressure.Cycle = "constant"
 crater.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-crater.Patch.y_upper.BCPressure.Type = 'FluxConst'
-crater.Patch.y_upper.BCPressure.Cycle = 'constant'
+crater.Patch.y_upper.BCPressure.Type = "FluxConst"
+crater.Patch.y_upper.BCPressure.Cycle = "constant"
 crater.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-crater.Patch.z_upper.BCPressure.Type = 'FluxConst'
-crater.Patch.z_upper.BCPressure.Cycle = 'constant'
+crater.Patch.z_upper.BCPressure.Type = "FluxConst"
+crater.Patch.z_upper.BCPressure.Cycle = "constant"
 crater.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-crater.TopoSlopesX.Type = 'Constant'
-crater.TopoSlopesX.GeomNames = 'domain'
+crater.TopoSlopesX.Type = "Constant"
+crater.TopoSlopesX.GeomNames = "domain"
 crater.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-crater.TopoSlopesY.Type = 'Constant'
-crater.TopoSlopesY.GeomNames = 'domain'
+crater.TopoSlopesY.Type = "Constant"
+crater.TopoSlopesY.GeomNames = "domain"
 crater.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-crater.Mannings.Type = 'Constant'
-crater.Mannings.GeomNames = 'domain'
-crater.Mannings.Geom.domain.Value = 0.
+crater.Mannings.Type = "Constant"
+crater.Mannings.GeomNames = "domain"
+crater.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-crater.ICPressure.Type = 'HydroStaticPatch'
-crater.ICPressure.GeomNames = 'domain'
+crater.ICPressure.Type = "HydroStaticPatch"
+crater.ICPressure.GeomNames = "domain"
 
 crater.Geom.domain.ICPressure.Value = 1.0
-crater.Geom.domain.ICPressure.RefPatch = 'z_lower'
-crater.Geom.domain.ICPressure.RefGeom = 'domain'
+crater.Geom.domain.ICPressure.RefPatch = "z_lower"
+crater.Geom.domain.ICPressure.RefGeom = "domain"
 
 crater.Geom.infiltration.ICPressure.Value = 10.0
-crater.Geom.infiltration.ICPressure.RefPatch = 'infiltration'
-crater.Geom.infiltration.ICPressure.RefGeom = 'domain'
+crater.Geom.infiltration.ICPressure.RefPatch = "infiltration"
+crater.Geom.infiltration.ICPressure.RefGeom = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.PhaseSources.water.Type = 'Constant'
-crater.PhaseSources.water.GeomNames = 'background'
+crater.PhaseSources.water.Type = "Constant"
+crater.PhaseSources.water.GeomNames = "background"
 crater.PhaseSources.water.Geom.background.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-crater.KnownSolution = 'NoKnownSolution'
+crater.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
-crater.Solver = 'Richards'
+# -----------------------------------------------------------------------------
+crater.Solver = "Richards"
 crater.Solver.MaxIter = 10000
 
 crater.Solver.Nonlinear.MaxIter = 15
@@ -481,12 +484,61 @@
 crater.Solver.Linear.KrylovDimension = 25
 crater.Solver.Linear.MaxRestarts = 10
 
-crater.Solver.Linear.Preconditioner = 'MGSemi'
+crater.Solver.Linear.Preconditioner = "MGSemi"
 crater.Solver.Linear.Preconditioner.MGSemi.MaxIter = 1
 crater.Solver.Linear.Preconditioner.MGSemi.MaxLevels = 100
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-crater.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = dir_name
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+crater.run(working_directory=new_output_dir_name)
+
+passed = True
+sig_digits = 5
+# Pressure was very close to zero and test was failing so ignore very small pressures even if
+# the numbers differ in sig_digits
+abs_value = 1e-200
+test_files = ["perm_x", "perm_y", "perm_z", "porosity"]
+
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+        sig_digits,
+    ):
+        passed = False
+
+
+for i in range(3):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/default_overland.pfmg.jac.py b/test/python/default_overland.pfmg.jac.py
index 672305bdd..bf2a000d1 100644
--- a/test/python/default_overland.pfmg.jac.py
+++ b/test/python/default_overland.pfmg.jac.py
@@ -1,24 +1,32 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  This runs the tilted-v catchment problem
 #  similar to that in Kollet and Maxwell (2006) AWR
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
+import sys, argparse
 from parflow import Run
-from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.fs import mkdir, get_absolute_path, rm
+from parflow.tools.compare import pf_test_file
 
-dover = Run("default_overland_pfmg_jac", __file__)
+run_name = "default_overland"
+dover = Run(run_name, __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+parser = argparse.ArgumentParser()
+parser.add_argument("-p", "--p", default=1)
+parser.add_argument("-q", "--q", default=1)
+parser.add_argument("-r", "--r", default=1)
+args = parser.parse_args()
 
 dover.FileVersion = 4
 
-dover.Process.Topology.P = 1
-dover.Process.Topology.Q = 1
-dover.Process.Topology.R = 1
+dover.Process.Topology.P = args.p
+dover.Process.Topology.Q = args.q
+dover.Process.Topology.R = args.r
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 dover.ComputationalGrid.Lower.X = 0.0
 dover.ComputationalGrid.Lower.Y = 0.0
@@ -30,27 +38,27 @@
 
 dover.ComputationalGrid.DX = 10.0
 dover.ComputationalGrid.DY = 10.0
-dover.ComputationalGrid.DZ = .05
+dover.ComputationalGrid.DZ = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.GeomInput.Names = 'domaininput leftinput rightinput channelinput'
+dover.GeomInput.Names = "domaininput leftinput rightinput channelinput"
 
-dover.GeomInput.domaininput.GeomName = 'domain'
-dover.GeomInput.leftinput.GeomName = 'left'
-dover.GeomInput.rightinput.GeomName = 'right'
-dover.GeomInput.channelinput.GeomName = 'channel'
+dover.GeomInput.domaininput.GeomName = "domain"
+dover.GeomInput.leftinput.GeomName = "left"
+dover.GeomInput.rightinput.GeomName = "right"
+dover.GeomInput.channelinput.GeomName = "channel"
 
-dover.GeomInput.domaininput.InputType = 'Box'
-dover.GeomInput.leftinput.InputType = 'Box'
-dover.GeomInput.rightinput.InputType = 'Box'
-dover.GeomInput.channelinput.InputType = 'Box'
+dover.GeomInput.domaininput.InputType = "Box"
+dover.GeomInput.leftinput.InputType = "Box"
+dover.GeomInput.rightinput.InputType = "Box"
+dover.GeomInput.channelinput.InputType = "Box"
 
-#---------------------------------------------------------
-# Domain Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Domain Geometry
+# ---------------------------------------------------------
 
 dover.Geom.domain.Lower.X = 0.0
 dover.Geom.domain.Lower.Y = 0.0
@@ -59,11 +67,11 @@
 dover.Geom.domain.Upper.X = 300.0
 dover.Geom.domain.Upper.Y = 300.0
 dover.Geom.domain.Upper.Z = 1.5
-dover.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+dover.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#---------------------------------------------------------
-# Left Slope Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Left Slope Geometry
+# ---------------------------------------------------------
 
 dover.Geom.left.Lower.X = 0.0
 dover.Geom.left.Lower.Y = 0.0
@@ -73,9 +81,9 @@
 dover.Geom.left.Upper.Y = 300.0
 dover.Geom.left.Upper.Z = 1.5
 
-#---------------------------------------------------------
-# Right Slope Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Right Slope Geometry
+# ---------------------------------------------------------
 
 dover.Geom.right.Lower.X = 160.0
 dover.Geom.right.Lower.Y = 0.0
@@ -85,9 +93,9 @@
 dover.Geom.right.Upper.Y = 300.0
 dover.Geom.right.Upper.Z = 1.5
 
-#---------------------------------------------------------
-# Channel Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Channel Geometry
+# ---------------------------------------------------------
 dover.Geom.channel.Lower.X = 140.0
 dover.Geom.channel.Lower.Y = 0.0
 dover.Geom.channel.Lower.Z = 0.0
@@ -96,11 +104,11 @@
 dover.Geom.channel.Upper.Y = 300.0
 dover.Geom.channel.Upper.Z = 1.5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Geom.Perm.Names = 'left right channel'
+dover.Geom.Perm.Names = "left right channel"
 
 # Values in m/hour
 
@@ -108,9 +116,9 @@
 # the following is ignored if the perm.type "Constant" settings are not
 # commented out, below.
 
-dover.Geom.left.Perm.Type = 'TurnBands'
-dover.Geom.left.Perm.LambdaX = 50.
-dover.Geom.left.Perm.LambdaY = 50.
+dover.Geom.left.Perm.Type = "TurnBands"
+dover.Geom.left.Perm.LambdaX = 50.0
+dover.Geom.left.Perm.LambdaY = 50.0
 dover.Geom.left.Perm.LambdaZ = 0.5
 dover.Geom.left.Perm.GeomMean = 0.01
 
@@ -120,12 +128,12 @@
 dover.Geom.left.Perm.KMax = 100.0
 dover.Geom.left.Perm.DelK = 0.2
 dover.Geom.left.Perm.Seed = 33333
-dover.Geom.left.Perm.LogNormal = 'Log'
-dover.Geom.left.Perm.StratType = 'Bottom'
+dover.Geom.left.Perm.LogNormal = "Log"
+dover.Geom.left.Perm.StratType = "Bottom"
 
-dover.Geom.right.Perm.Type = 'TurnBands'
-dover.Geom.right.Perm.LambdaX = 50.
-dover.Geom.right.Perm.LambdaY = 50.
+dover.Geom.right.Perm.Type = "TurnBands"
+dover.Geom.right.Perm.LambdaX = 50.0
+dover.Geom.right.Perm.LambdaY = 50.0
 dover.Geom.right.Perm.LambdaZ = 0.5
 dover.Geom.right.Perm.GeomMean = 0.05
 
@@ -135,8 +143,8 @@
 dover.Geom.right.Perm.KMax = 100.0
 dover.Geom.right.Perm.DelK = 0.2
 dover.Geom.right.Perm.Seed = 13333
-dover.Geom.right.Perm.LogNormal = 'Log'
-dover.Geom.right.Perm.StratType = 'Bottom'
+dover.Geom.right.Perm.LogNormal = "Log"
+dover.Geom.right.Perm.StratType = "Bottom"
 
 # hydraulic conductivity is very low, but not zero, top node will have to saturate
 # before overland flow can begin and will be driven by hortonian flow
@@ -144,254 +152,289 @@
 # turning bands above.  Run time increases quite a bit with a heterogeneous
 # subsurface
 
-dover.Geom.left.Perm.Type = 'Constant'
+dover.Geom.left.Perm.Type = "Constant"
 dover.Geom.left.Perm.Value = 0.001
 
-dover.Geom.right.Perm.Type = 'Constant'
+dover.Geom.right.Perm.Type = "Constant"
 dover.Geom.right.Perm.Value = 0.01
 
-dover.Geom.channel.Perm.Type = 'Constant'
+dover.Geom.channel.Perm.Type = "Constant"
 dover.Geom.channel.Perm.Value = 0.00001
 
-dover.Perm.TensorType = 'TensorByGeom'
+dover.Perm.TensorType = "TensorByGeom"
 
-dover.Geom.Perm.TensorByGeom.Names = 'domain'
+dover.Geom.Perm.TensorByGeom.Names = "domain"
 
 dover.Geom.domain.Perm.TensorValX = 1.0
 dover.Geom.domain.Perm.TensorValY = 1.0
 dover.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.SpecificStorage.Type = 'Constant'
-dover.SpecificStorage.GeomNames = 'domain'
+dover.SpecificStorage.Type = "Constant"
+dover.SpecificStorage.GeomNames = "domain"
 dover.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Phase.Names = 'water'
+dover.Phase.Names = "water"
 
-dover.Phase.water.Density.Type = 'Constant'
+dover.Phase.water.Density.Type = "Constant"
 dover.Phase.water.Density.Value = 1.0
 
-dover.Phase.water.Viscosity.Type = 'Constant'
+dover.Phase.water.Viscosity.Type = "Constant"
 dover.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Contaminants.Names = ''
+dover.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Geom.Retardation.GeomNames = ''
+dover.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dover.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dover.TimingInfo.BaseUnit = 0.1
 dover.TimingInfo.StartCount = 0
 dover.TimingInfo.StartTime = 0.0
 dover.TimingInfo.StopTime = 0.4
 dover.TimingInfo.DumpInterval = -1
-dover.TimeStep.Type = 'Constant'
+dover.TimeStep.Type = "Constant"
 dover.TimeStep.Value = 0.1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Geom.Porosity.GeomNames = 'left right channel'
+dover.Geom.Porosity.GeomNames = "left right channel"
 
-dover.Geom.left.Porosity.Type = 'Constant'
+dover.Geom.left.Porosity.Type = "Constant"
 dover.Geom.left.Porosity.Value = 0.25
 
-dover.Geom.right.Porosity.Type = 'Constant'
+dover.Geom.right.Porosity.Type = "Constant"
 dover.Geom.right.Porosity.Value = 0.25
 
-dover.Geom.channel.Porosity.Type = 'Constant'
+dover.Geom.channel.Porosity.Type = "Constant"
 dover.Geom.channel.Porosity.Value = 0.01
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Domain.GeomName = 'domain'
+dover.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Phase.RelPerm.Type = 'VanGenuchten'
-dover.Phase.RelPerm.GeomNames = 'domain'
+dover.Phase.RelPerm.Type = "VanGenuchten"
+dover.Phase.RelPerm.GeomNames = "domain"
 
 dover.Geom.domain.RelPerm.Alpha = 6.0
-dover.Geom.domain.RelPerm.N = 2.
+dover.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.Phase.Saturation.Type = 'VanGenuchten'
-dover.Phase.Saturation.GeomNames = 'domain'
+dover.Phase.Saturation.Type = "VanGenuchten"
+dover.Phase.Saturation.GeomNames = "domain"
 
 dover.Geom.domain.Saturation.Alpha = 6.0
-dover.Geom.domain.Saturation.N = 2.
+dover.Geom.domain.Saturation.N = 2.0
 dover.Geom.domain.Saturation.SRes = 0.2
 dover.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Wells.Names = ''
+dover.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Cycle.Names = 'constant rainrec'
-dover.Cycle.constant.Names = 'alltime'
+dover.Cycle.Names = "constant rainrec"
+dover.Cycle.constant.Names = "alltime"
 dover.Cycle.constant.alltime.Length = 1
 dover.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-dover.Cycle.rainrec.Names = 'rain rec'
+dover.Cycle.rainrec.Names = "rain rec"
 dover.Cycle.rainrec.rain.Length = 1
 dover.Cycle.rainrec.rec.Length = 2
 dover.Cycle.rainrec.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dover.BCPressure.PatchNames = dover.Geom.domain.Patches
 
-dover.Patch.x_lower.BCPressure.Type = 'FluxConst'
-dover.Patch.x_lower.BCPressure.Cycle = 'constant'
+dover.Patch.x_lower.BCPressure.Type = "FluxConst"
+dover.Patch.x_lower.BCPressure.Cycle = "constant"
 dover.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-dover.Patch.y_lower.BCPressure.Type = 'FluxConst'
-dover.Patch.y_lower.BCPressure.Cycle = 'constant'
+dover.Patch.y_lower.BCPressure.Type = "FluxConst"
+dover.Patch.y_lower.BCPressure.Cycle = "constant"
 dover.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-dover.Patch.z_lower.BCPressure.Type = 'FluxConst'
-dover.Patch.z_lower.BCPressure.Cycle = 'constant'
+dover.Patch.z_lower.BCPressure.Type = "FluxConst"
+dover.Patch.z_lower.BCPressure.Cycle = "constant"
 dover.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-dover.Patch.x_upper.BCPressure.Type = 'FluxConst'
-dover.Patch.x_upper.BCPressure.Cycle = 'constant'
+dover.Patch.x_upper.BCPressure.Type = "FluxConst"
+dover.Patch.x_upper.BCPressure.Cycle = "constant"
 dover.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-dover.Patch.y_upper.BCPressure.Type = 'FluxConst'
-dover.Patch.y_upper.BCPressure.Cycle = 'constant'
+dover.Patch.y_upper.BCPressure.Type = "FluxConst"
+dover.Patch.y_upper.BCPressure.Cycle = "constant"
 dover.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
 ## overland flow boundary condition with very heavy rainfall then slight ET
-dover.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-dover.Patch.z_upper.BCPressure.Cycle = 'rainrec'
+dover.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+dover.Patch.z_upper.BCPressure.Cycle = "rainrec"
 dover.Patch.z_upper.BCPressure.rain.Value = -0.05
 dover.Patch.z_upper.BCPressure.rec.Value = 0.000001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.TopoSlopesX.Type = 'Constant'
-dover.TopoSlopesX.GeomNames = 'left right channel'
+dover.TopoSlopesX.Type = "Constant"
+dover.TopoSlopesX.GeomNames = "left right channel"
 dover.TopoSlopesX.Geom.left.Value = -0.005
 dover.TopoSlopesX.Geom.right.Value = 0.005
 dover.TopoSlopesX.Geom.channel.Value = 0.00
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.TopoSlopesY.Type = 'Constant'
-dover.TopoSlopesY.GeomNames = 'left right channel'
+dover.TopoSlopesY.Type = "Constant"
+dover.TopoSlopesY.GeomNames = "left right channel"
 dover.TopoSlopesY.Geom.left.Value = 0.001
 dover.TopoSlopesY.Geom.right.Value = 0.001
 dover.TopoSlopesY.Geom.channel.Value = 0.001
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-dover.Mannings.Type = 'Constant'
-dover.Mannings.GeomNames = 'left right channel'
-dover.Mannings.Geom.left.Value = 5.e-6
-dover.Mannings.Geom.right.Value = 5.e-6
-dover.Mannings.Geom.channel.Value = 1.e-6
+dover.Mannings.Type = "Constant"
+dover.Mannings.GeomNames = "left right channel"
+dover.Mannings.Geom.left.Value = 5.0e-6
+dover.Mannings.Geom.right.Value = 5.0e-6
+dover.Mannings.Geom.channel.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.PhaseSources.water.Type = 'Constant'
-dover.PhaseSources.water.GeomNames = 'domain'
+dover.PhaseSources.water.Type = "Constant"
+dover.PhaseSources.water.GeomNames = "domain"
 dover.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.KnownSolution = 'NoKnownSolution'
+dover.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Solver = 'Richards'
+dover.Solver = "Richards"
 dover.Solver.MaxIter = 2500
 
 dover.Solver.Nonlinear.MaxIter = 20
 dover.Solver.Nonlinear.ResidualTol = 1e-9
-dover.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+dover.Solver.Nonlinear.EtaChoice = "EtaConstant"
 dover.Solver.Nonlinear.EtaValue = 0.01
 dover.Solver.Nonlinear.UseJacobian = True
 dover.Solver.Nonlinear.DerivativeEpsilon = 1e-8
 dover.Solver.Nonlinear.StepTol = 1e-20
-dover.Solver.Nonlinear.Globalization = 'LineSearch'
+dover.Solver.Nonlinear.Globalization = "LineSearch"
 dover.Solver.Linear.KrylovDimension = 20
 dover.Solver.Linear.MaxRestart = 2
 
-dover.Solver.Linear.Preconditioner = 'PFMG'
+dover.Solver.Linear.Preconditioner = "PFMG"
 dover.Solver.PrintSubsurf = False
-dover.Solver.Drop = 1E-20
-dover.Solver.AbsTol = 1E-9
+dover.Solver.Drop = 1e-20
+dover.Solver.AbsTol = 1e-9
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be at the bottom of the domain, the top layer is initially dry
-dover.ICPressure.Type = 'HydroStaticPatch'
-dover.ICPressure.GeomNames = 'domain'
+dover.ICPressure.Type = "HydroStaticPatch"
+dover.ICPressure.GeomNames = "domain"
 dover.Geom.domain.ICPressure.Value = -3.0
 
-dover.Geom.domain.ICPressure.RefGeom = 'domain'
-dover.Geom.domain.ICPressure.RefPatch = 'z_upper'
+dover.Geom.domain.ICPressure.RefGeom = "domain"
+dover.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/dover_pj')
-mkdir(dir_name)
-dover.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/default_overland_pfmg_jac")
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+dover.run(working_directory=new_output_dir_name)
+
+passed = True
+sig_digits = 5
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+        sig_digits,
+    ):
+        passed = False
+
+test_files = ["press", "satur"]
+for i in range(5):
+    for test_file in test_files:
+        timestep = str(i).rjust(5, "0")
+        filename = f"/{run_name}.out.{test_file}.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in {test_file} for timestep {timestep}",
+            sig_digits,
+        ):
+            passed = False
+
+rm(new_output_dir_name)
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/default_overland.pfmg_octree.fulljac.py b/test/python/default_overland.pfmg_octree.fulljac.py
index 012f91f57..953ae980a 100644
--- a/test/python/default_overland.pfmg_octree.fulljac.py
+++ b/test/python/default_overland.pfmg_octree.fulljac.py
@@ -1,24 +1,33 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  This runs the tilted-v catchment problem
 #  similar to that in Kollet and Maxwell (2006) AWR
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
+import sys, argparse
 from parflow import Run
-from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.fs import mkdir, get_absolute_path, rm
+from parflow.tools.compare import pf_test_file
 
-dover = Run("default_overland_pfmg_octree_fulljac", __file__)
+run_name = "default_overland"
+dover = Run(run_name, __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 dover.FileVersion = 4
 
-dover.Process.Topology.P = 1
-dover.Process.Topology.Q = 1
-dover.Process.Topology.R = 1
+parser = argparse.ArgumentParser()
+parser.add_argument("-p", "--p", default=1)
+parser.add_argument("-q", "--q", default=1)
+parser.add_argument("-r", "--r", default=1)
+args = parser.parse_args()
 
-#---------------------------------------------------------
+dover.Process.Topology.P = args.p
+dover.Process.Topology.Q = args.q
+dover.Process.Topology.R = args.r
+
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 dover.ComputationalGrid.Lower.X = 0.0
 dover.ComputationalGrid.Lower.Y = 0.0
@@ -30,27 +39,27 @@
 
 dover.ComputationalGrid.DX = 10.0
 dover.ComputationalGrid.DY = 10.0
-dover.ComputationalGrid.DZ = .05
+dover.ComputationalGrid.DZ = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.GeomInput.Names = 'domaininput leftinput rightinput channelinput'
+dover.GeomInput.Names = "domaininput leftinput rightinput channelinput"
 
-dover.GeomInput.domaininput.GeomName = 'domain'
-dover.GeomInput.leftinput.GeomName = 'left'
-dover.GeomInput.rightinput.GeomName = 'right'
-dover.GeomInput.channelinput.GeomName = 'channel'
+dover.GeomInput.domaininput.GeomName = "domain"
+dover.GeomInput.leftinput.GeomName = "left"
+dover.GeomInput.rightinput.GeomName = "right"
+dover.GeomInput.channelinput.GeomName = "channel"
 
-dover.GeomInput.domaininput.InputType = 'Box'
-dover.GeomInput.leftinput.InputType = 'Box'
-dover.GeomInput.rightinput.InputType = 'Box'
-dover.GeomInput.channelinput.InputType = 'Box'
+dover.GeomInput.domaininput.InputType = "Box"
+dover.GeomInput.leftinput.InputType = "Box"
+dover.GeomInput.rightinput.InputType = "Box"
+dover.GeomInput.channelinput.InputType = "Box"
 
-#---------------------------------------------------------
-# Domain Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Domain Geometry
+# ---------------------------------------------------------
 
 dover.Geom.domain.Lower.X = 0.0
 dover.Geom.domain.Lower.Y = 0.0
@@ -59,11 +68,11 @@
 dover.Geom.domain.Upper.X = 300.0
 dover.Geom.domain.Upper.Y = 300.0
 dover.Geom.domain.Upper.Z = 1.5
-dover.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+dover.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#---------------------------------------------------------
-# Left Slope Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Left Slope Geometry
+# ---------------------------------------------------------
 
 dover.Geom.left.Lower.X = 0.0
 dover.Geom.left.Lower.Y = 0.0
@@ -73,9 +82,9 @@
 dover.Geom.left.Upper.Y = 300.0
 dover.Geom.left.Upper.Z = 1.5
 
-#---------------------------------------------------------
-# Right Slope Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Right Slope Geometry
+# ---------------------------------------------------------
 
 dover.Geom.right.Lower.X = 160.0
 dover.Geom.right.Lower.Y = 0.0
@@ -85,9 +94,9 @@
 dover.Geom.right.Upper.Y = 300.0
 dover.Geom.right.Upper.Z = 1.5
 
-#---------------------------------------------------------
-# Channel Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Channel Geometry
+# ---------------------------------------------------------
 
 dover.Geom.channel.Lower.X = 140.0
 dover.Geom.channel.Lower.Y = 0.0
@@ -97,11 +106,11 @@
 dover.Geom.channel.Upper.Y = 300.0
 dover.Geom.channel.Upper.Z = 1.5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Geom.Perm.Names = 'left right channel'
+dover.Geom.Perm.Names = "left right channel"
 
 # Values in m/hour
 
@@ -109,9 +118,9 @@
 # the following is ignored if the perm.type "Constant" settings are not
 # commented out, below.
 
-dover.Geom.left.Perm.Type = 'TurnBands'
-dover.Geom.left.Perm.LambdaX = 50.
-dover.Geom.left.Perm.LambdaY = 50.
+dover.Geom.left.Perm.Type = "TurnBands"
+dover.Geom.left.Perm.LambdaX = 50.0
+dover.Geom.left.Perm.LambdaY = 50.0
 dover.Geom.left.Perm.LambdaZ = 0.5
 dover.Geom.left.Perm.GeomMean = 0.01
 
@@ -121,12 +130,12 @@
 dover.Geom.left.Perm.KMax = 100.0
 dover.Geom.left.Perm.DelK = 0.2
 dover.Geom.left.Perm.Seed = 33333
-dover.Geom.left.Perm.LogNormal = 'Log'
-dover.Geom.left.Perm.StratType = 'Bottom'
+dover.Geom.left.Perm.LogNormal = "Log"
+dover.Geom.left.Perm.StratType = "Bottom"
 
-dover.Geom.right.Perm.Type = 'TurnBands'
-dover.Geom.right.Perm.LambdaX = 50.
-dover.Geom.right.Perm.LambdaY = 50.
+dover.Geom.right.Perm.Type = "TurnBands"
+dover.Geom.right.Perm.LambdaX = 50.0
+dover.Geom.right.Perm.LambdaY = 50.0
 dover.Geom.right.Perm.LambdaZ = 0.5
 dover.Geom.right.Perm.GeomMean = 0.05
 
@@ -136,8 +145,8 @@
 dover.Geom.right.Perm.KMax = 100.0
 dover.Geom.right.Perm.DelK = 0.2
 dover.Geom.right.Perm.Seed = 13333
-dover.Geom.right.Perm.LogNormal = 'Log'
-dover.Geom.right.Perm.StratType = 'Bottom'
+dover.Geom.right.Perm.LogNormal = "Log"
+dover.Geom.right.Perm.StratType = "Bottom"
 
 # hydraulic conductivity is very low, but not zero, top node will have to saturate
 # before overland flow can begin and will be driven by hortonian flow
@@ -145,255 +154,292 @@
 # turning bands above.  Run time increases quite a bit with a heterogeneous
 # subsurface
 
-dover.Geom.left.Perm.Type = 'Constant'
+dover.Geom.left.Perm.Type = "Constant"
 dover.Geom.left.Perm.Value = 0.001
 
-dover.Geom.right.Perm.Type = 'Constant'
+dover.Geom.right.Perm.Type = "Constant"
 dover.Geom.right.Perm.Value = 0.01
 
-dover.Geom.channel.Perm.Type = 'Constant'
+dover.Geom.channel.Perm.Type = "Constant"
 dover.Geom.channel.Perm.Value = 0.00001
 
-dover.Perm.TensorType = 'TensorByGeom'
+dover.Perm.TensorType = "TensorByGeom"
 
-dover.Geom.Perm.TensorByGeom.Names = 'domain'
+dover.Geom.Perm.TensorByGeom.Names = "domain"
 
 dover.Geom.domain.Perm.TensorValX = 1.0
 dover.Geom.domain.Perm.TensorValY = 1.0
 dover.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.SpecificStorage.Type = 'Constant'
-dover.SpecificStorage.GeomNames = 'domain'
+dover.SpecificStorage.Type = "Constant"
+dover.SpecificStorage.GeomNames = "domain"
 dover.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Phase.Names = 'water'
+dover.Phase.Names = "water"
 
-dover.Phase.water.Density.Type = 'Constant'
+dover.Phase.water.Density.Type = "Constant"
 dover.Phase.water.Density.Value = 1.0
 
-dover.Phase.water.Viscosity.Type = 'Constant'
+dover.Phase.water.Viscosity.Type = "Constant"
 dover.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Contaminants.Names = ''
+dover.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Geom.Retardation.GeomNames = ''
+dover.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dover.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dover.TimingInfo.BaseUnit = 0.1
 dover.TimingInfo.StartCount = 0
 dover.TimingInfo.StartTime = 0.0
 dover.TimingInfo.StopTime = 0.4
 dover.TimingInfo.DumpInterval = -1
-dover.TimeStep.Type = 'Constant'
+dover.TimeStep.Type = "Constant"
 dover.TimeStep.Value = 0.1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Geom.Porosity.GeomNames = 'left right channel'
+dover.Geom.Porosity.GeomNames = "left right channel"
 
-dover.Geom.left.Porosity.Type = 'Constant'
+dover.Geom.left.Porosity.Type = "Constant"
 dover.Geom.left.Porosity.Value = 0.25
 
-dover.Geom.right.Porosity.Type = 'Constant'
+dover.Geom.right.Porosity.Type = "Constant"
 dover.Geom.right.Porosity.Value = 0.25
 
-dover.Geom.channel.Porosity.Type = 'Constant'
+dover.Geom.channel.Porosity.Type = "Constant"
 dover.Geom.channel.Porosity.Value = 0.01
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Domain.GeomName = 'domain'
+dover.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Phase.RelPerm.Type = 'VanGenuchten'
-dover.Phase.RelPerm.GeomNames = 'domain'
+dover.Phase.RelPerm.Type = "VanGenuchten"
+dover.Phase.RelPerm.GeomNames = "domain"
 
 dover.Geom.domain.RelPerm.Alpha = 6.0
-dover.Geom.domain.RelPerm.N = 2.
+dover.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.Phase.Saturation.Type = 'VanGenuchten'
-dover.Phase.Saturation.GeomNames = 'domain'
+dover.Phase.Saturation.Type = "VanGenuchten"
+dover.Phase.Saturation.GeomNames = "domain"
 
 dover.Geom.domain.Saturation.Alpha = 6.0
-dover.Geom.domain.Saturation.N = 2.
+dover.Geom.domain.Saturation.N = 2.0
 dover.Geom.domain.Saturation.SRes = 0.2
 dover.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Wells.Names = ''
+dover.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Cycle.Names = 'constant rainrec'
-dover.Cycle.constant.Names = 'alltime'
+dover.Cycle.Names = "constant rainrec"
+dover.Cycle.constant.Names = "alltime"
 dover.Cycle.constant.alltime.Length = 1
 dover.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-dover.Cycle.rainrec.Names = 'rain rec'
+dover.Cycle.rainrec.Names = "rain rec"
 dover.Cycle.rainrec.rain.Length = 1
 dover.Cycle.rainrec.rec.Length = 2
 dover.Cycle.rainrec.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dover.BCPressure.PatchNames = dover.Geom.domain.Patches
 
-dover.Patch.x_lower.BCPressure.Type = 'FluxConst'
-dover.Patch.x_lower.BCPressure.Cycle = 'constant'
+dover.Patch.x_lower.BCPressure.Type = "FluxConst"
+dover.Patch.x_lower.BCPressure.Cycle = "constant"
 dover.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-dover.Patch.y_lower.BCPressure.Type = 'FluxConst'
-dover.Patch.y_lower.BCPressure.Cycle = 'constant'
+dover.Patch.y_lower.BCPressure.Type = "FluxConst"
+dover.Patch.y_lower.BCPressure.Cycle = "constant"
 dover.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-dover.Patch.z_lower.BCPressure.Type = 'FluxConst'
-dover.Patch.z_lower.BCPressure.Cycle = 'constant'
+dover.Patch.z_lower.BCPressure.Type = "FluxConst"
+dover.Patch.z_lower.BCPressure.Cycle = "constant"
 dover.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-dover.Patch.x_upper.BCPressure.Type = 'FluxConst'
-dover.Patch.x_upper.BCPressure.Cycle = 'constant'
+dover.Patch.x_upper.BCPressure.Type = "FluxConst"
+dover.Patch.x_upper.BCPressure.Cycle = "constant"
 dover.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-dover.Patch.y_upper.BCPressure.Type = 'FluxConst'
-dover.Patch.y_upper.BCPressure.Cycle = 'constant'
+dover.Patch.y_upper.BCPressure.Type = "FluxConst"
+dover.Patch.y_upper.BCPressure.Cycle = "constant"
 dover.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
 ## overland flow boundary condition with very heavy rainfall then slight ET
-dover.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-dover.Patch.z_upper.BCPressure.Cycle = 'rainrec'
+dover.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+dover.Patch.z_upper.BCPressure.Cycle = "rainrec"
 dover.Patch.z_upper.BCPressure.rain.Value = -0.05
 dover.Patch.z_upper.BCPressure.rec.Value = 0.000001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.TopoSlopesX.Type = 'Constant'
-dover.TopoSlopesX.GeomNames = 'left right channel'
+dover.TopoSlopesX.Type = "Constant"
+dover.TopoSlopesX.GeomNames = "left right channel"
 dover.TopoSlopesX.Geom.left.Value = -0.005
 dover.TopoSlopesX.Geom.right.Value = 0.005
 dover.TopoSlopesX.Geom.channel.Value = 0.00
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.TopoSlopesY.Type = 'Constant'
-dover.TopoSlopesY.GeomNames = 'left right channel'
+dover.TopoSlopesY.Type = "Constant"
+dover.TopoSlopesY.GeomNames = "left right channel"
 dover.TopoSlopesY.Geom.left.Value = 0.001
 dover.TopoSlopesY.Geom.right.Value = 0.001
 dover.TopoSlopesY.Geom.channel.Value = 0.001
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-dover.Mannings.Type = 'Constant'
-dover.Mannings.GeomNames = 'left right channel'
-dover.Mannings.Geom.left.Value = 5.e-6
-dover.Mannings.Geom.right.Value = 5.e-6
-dover.Mannings.Geom.channel.Value = 1.e-6
+dover.Mannings.Type = "Constant"
+dover.Mannings.GeomNames = "left right channel"
+dover.Mannings.Geom.left.Value = 5.0e-6
+dover.Mannings.Geom.right.Value = 5.0e-6
+dover.Mannings.Geom.channel.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.PhaseSources.water.Type = 'Constant'
-dover.PhaseSources.water.GeomNames = 'domain'
+dover.PhaseSources.water.Type = "Constant"
+dover.PhaseSources.water.GeomNames = "domain"
 dover.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.KnownSolution = 'NoKnownSolution'
+dover.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Solver = 'Richards'
+dover.Solver = "Richards"
 dover.Solver.MaxIter = 2500
 
 dover.Solver.Nonlinear.MaxIter = 20
 dover.Solver.Nonlinear.ResidualTol = 1e-9
-dover.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+dover.Solver.Nonlinear.EtaChoice = "EtaConstant"
 dover.Solver.Nonlinear.EtaValue = 0.01
 dover.Solver.Nonlinear.UseJacobian = True
 dover.Solver.Nonlinear.DerivativeEpsilon = 1e-8
 dover.Solver.Nonlinear.StepTol = 1e-20
-dover.Solver.Nonlinear.Globalization = 'LineSearch'
+dover.Solver.Nonlinear.Globalization = "LineSearch"
 dover.Solver.Linear.KrylovDimension = 20
 dover.Solver.Linear.MaxRestart = 2
 
-dover.Solver.Linear.Preconditioner = 'PFMGOctree'
-dover.Solver.Linear.Preconditioner.PCMatrixType = 'FullJacobian'
+dover.Solver.Linear.Preconditioner = "PFMGOctree"
+dover.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
 dover.Solver.PrintSubsurf = False
-dover.Solver.Drop = 1E-20
-dover.Solver.AbsTol = 1E-9
+dover.Solver.Drop = 1e-20
+dover.Solver.AbsTol = 1e-9
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be at the bottom of the domain, the top layer is initially dry
-dover.ICPressure.Type = 'HydroStaticPatch'
-dover.ICPressure.GeomNames = 'domain'
+dover.ICPressure.Type = "HydroStaticPatch"
+dover.ICPressure.GeomNames = "domain"
 dover.Geom.domain.ICPressure.Value = -3.0
 
-dover.Geom.domain.ICPressure.RefGeom = 'domain'
-dover.Geom.domain.ICPressure.RefPatch = 'z_upper'
+dover.Geom.domain.ICPressure.RefGeom = "domain"
+dover.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/dover_pof')
-mkdir(dir_name)
-dover.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path(
+    "test_output/default_overland_pfmg_octree_fulljac"
+)
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+dover.run(working_directory=new_output_dir_name)
+
+passed = True
+sig_digits = 5
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+        sig_digits,
+    ):
+        passed = False
+
+test_files = ["press", "satur"]
+for i in range(5):
+    for test_file in test_files:
+        timestep = str(i).rjust(5, "0")
+        filename = f"/{run_name}.out.{test_file}.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in {test_file} for timestep {timestep}",
+            sig_digits,
+        ):
+            passed = False
+
+rm(new_output_dir_name)
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/default_overland.pfmg_octree.jac.py b/test/python/default_overland.pfmg_octree.jac.py
index b7e6d9bcf..af8078c71 100644
--- a/test/python/default_overland.pfmg_octree.jac.py
+++ b/test/python/default_overland.pfmg_octree.jac.py
@@ -1,14 +1,14 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  This runs the tilted-v catchment problem
 #  similar to that in Kollet and Maxwell (2006) AWR
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
 
 dover = Run("default_overland_pfmg_octree_jac", __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 dover.FileVersion = 4
 
@@ -16,9 +16,9 @@
 dover.Process.Topology.Q = 1
 dover.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 dover.ComputationalGrid.Lower.X = 0.0
 dover.ComputationalGrid.Lower.Y = 0.0
@@ -30,27 +30,27 @@
 
 dover.ComputationalGrid.DX = 10.0
 dover.ComputationalGrid.DY = 10.0
-dover.ComputationalGrid.DZ = .05
+dover.ComputationalGrid.DZ = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.GeomInput.Names = 'domaininput leftinput rightinput channelinput'
+dover.GeomInput.Names = "domaininput leftinput rightinput channelinput"
 
-dover.GeomInput.domaininput.GeomName = 'domain'
-dover.GeomInput.leftinput.GeomName = 'left'
-dover.GeomInput.rightinput.GeomName = 'right'
-dover.GeomInput.channelinput.GeomName = 'channel'
+dover.GeomInput.domaininput.GeomName = "domain"
+dover.GeomInput.leftinput.GeomName = "left"
+dover.GeomInput.rightinput.GeomName = "right"
+dover.GeomInput.channelinput.GeomName = "channel"
 
-dover.GeomInput.domaininput.InputType = 'Box'
-dover.GeomInput.leftinput.InputType = 'Box'
-dover.GeomInput.rightinput.InputType = 'Box'
-dover.GeomInput.channelinput.InputType = 'Box'
+dover.GeomInput.domaininput.InputType = "Box"
+dover.GeomInput.leftinput.InputType = "Box"
+dover.GeomInput.rightinput.InputType = "Box"
+dover.GeomInput.channelinput.InputType = "Box"
 
-#---------------------------------------------------------
-# Domain Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Domain Geometry
+# ---------------------------------------------------------
 
 dover.Geom.domain.Lower.X = 0.0
 dover.Geom.domain.Lower.Y = 0.0
@@ -59,11 +59,11 @@
 dover.Geom.domain.Upper.X = 300.0
 dover.Geom.domain.Upper.Y = 300.0
 dover.Geom.domain.Upper.Z = 1.5
-dover.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+dover.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#---------------------------------------------------------
-# Left Slope Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Left Slope Geometry
+# ---------------------------------------------------------
 
 dover.Geom.left.Lower.X = 0.0
 dover.Geom.left.Lower.Y = 0.0
@@ -73,9 +73,9 @@
 dover.Geom.left.Upper.Y = 300.0
 dover.Geom.left.Upper.Z = 1.5
 
-#---------------------------------------------------------
-# Right Slope Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Right Slope Geometry
+# ---------------------------------------------------------
 
 dover.Geom.right.Lower.X = 160.0
 dover.Geom.right.Lower.Y = 0.0
@@ -85,9 +85,9 @@
 dover.Geom.right.Upper.Y = 300.0
 dover.Geom.right.Upper.Z = 1.5
 
-#---------------------------------------------------------
-# Channel Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Channel Geometry
+# ---------------------------------------------------------
 
 dover.Geom.channel.Lower.X = 140.0
 dover.Geom.channel.Lower.Y = 0.0
@@ -97,11 +97,11 @@
 dover.Geom.channel.Upper.Y = 300.0
 dover.Geom.channel.Upper.Z = 1.5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Geom.Perm.Names = 'left right channel'
+dover.Geom.Perm.Names = "left right channel"
 
 # Values in m/hour
 
@@ -109,9 +109,9 @@
 # the following is ignored if the perm.type "Constant" settings are not
 # commented out, below.
 
-dover.Geom.left.Perm.Type = 'TurnBands'
-dover.Geom.left.Perm.LambdaX = 50.
-dover.Geom.left.Perm.LambdaY = 50.
+dover.Geom.left.Perm.Type = "TurnBands"
+dover.Geom.left.Perm.LambdaX = 50.0
+dover.Geom.left.Perm.LambdaY = 50.0
 dover.Geom.left.Perm.LambdaZ = 0.5
 dover.Geom.left.Perm.GeomMean = 0.01
 
@@ -121,12 +121,12 @@
 dover.Geom.left.Perm.KMax = 100.0
 dover.Geom.left.Perm.DelK = 0.2
 dover.Geom.left.Perm.Seed = 33333
-dover.Geom.left.Perm.LogNormal = 'Log'
-dover.Geom.left.Perm.StratType = 'Bottom'
+dover.Geom.left.Perm.LogNormal = "Log"
+dover.Geom.left.Perm.StratType = "Bottom"
 
-dover.Geom.right.Perm.Type = 'TurnBands'
-dover.Geom.right.Perm.LambdaX = 50.
-dover.Geom.right.Perm.LambdaY = 50.
+dover.Geom.right.Perm.Type = "TurnBands"
+dover.Geom.right.Perm.LambdaX = 50.0
+dover.Geom.right.Perm.LambdaY = 50.0
 dover.Geom.right.Perm.LambdaZ = 0.5
 dover.Geom.right.Perm.GeomMean = 0.05
 
@@ -136,8 +136,8 @@
 dover.Geom.right.Perm.KMax = 100.0
 dover.Geom.right.Perm.DelK = 0.2
 dover.Geom.right.Perm.Seed = 13333
-dover.Geom.right.Perm.LogNormal = 'Log'
-dover.Geom.right.Perm.StratType = 'Bottom'
+dover.Geom.right.Perm.LogNormal = "Log"
+dover.Geom.right.Perm.StratType = "Bottom"
 
 # hydraulic conductivity is very low, but not zero, top node will have to saturate
 # before overland flow can begin and will be driven by hortonian flow
@@ -145,259 +145,259 @@
 # turning bands above.  Run time increases quite a bit with a heterogeneous
 # subsurface
 
-dover.Geom.left.Perm.Type = 'Constant'
+dover.Geom.left.Perm.Type = "Constant"
 dover.Geom.left.Perm.Value = 0.001
 
-dover.Geom.right.Perm.Type = 'Constant'
+dover.Geom.right.Perm.Type = "Constant"
 dover.Geom.right.Perm.Value = 0.01
 
-dover.Geom.channel.Perm.Type = 'Constant'
+dover.Geom.channel.Perm.Type = "Constant"
 dover.Geom.channel.Perm.Value = 0.00001
 
-dover.Perm.TensorType = 'TensorByGeom'
+dover.Perm.TensorType = "TensorByGeom"
 
-dover.Geom.Perm.TensorByGeom.Names = 'domain'
+dover.Geom.Perm.TensorByGeom.Names = "domain"
 
 dover.Geom.domain.Perm.TensorValX = 1.0
 dover.Geom.domain.Perm.TensorValY = 1.0
 dover.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.SpecificStorage.Type = 'Constant'
-dover.SpecificStorage.GeomNames = 'domain'
+dover.SpecificStorage.Type = "Constant"
+dover.SpecificStorage.GeomNames = "domain"
 dover.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Phase.Names = 'water'
+dover.Phase.Names = "water"
 
-dover.Phase.water.Density.Type = 'Constant'
+dover.Phase.water.Density.Type = "Constant"
 dover.Phase.water.Density.Value = 1.0
 
-dover.Phase.water.Viscosity.Type = 'Constant'
+dover.Phase.water.Viscosity.Type = "Constant"
 dover.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Contaminants.Names = ''
+dover.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Geom.Retardation.GeomNames = ''
+dover.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dover.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dover.TimingInfo.BaseUnit = 0.1
 dover.TimingInfo.StartCount = 0
 dover.TimingInfo.StartTime = 0.0
 dover.TimingInfo.StopTime = 0.4
 dover.TimingInfo.DumpInterval = -1
-dover.TimeStep.Type = 'Constant'
+dover.TimeStep.Type = "Constant"
 dover.TimeStep.Value = 0.1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Geom.Porosity.GeomNames = 'left right channel'
+dover.Geom.Porosity.GeomNames = "left right channel"
 
-dover.Geom.left.Porosity.Type = 'Constant'
+dover.Geom.left.Porosity.Type = "Constant"
 dover.Geom.left.Porosity.Value = 0.25
 
-dover.Geom.right.Porosity.Type = 'Constant'
+dover.Geom.right.Porosity.Type = "Constant"
 dover.Geom.right.Porosity.Value = 0.25
 
-dover.Geom.channel.Porosity.Type = 'Constant'
+dover.Geom.channel.Porosity.Type = "Constant"
 dover.Geom.channel.Porosity.Value = 0.01
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Domain.GeomName = 'domain'
+dover.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Phase.RelPerm.Type = 'VanGenuchten'
-dover.Phase.RelPerm.GeomNames = 'domain'
+dover.Phase.RelPerm.Type = "VanGenuchten"
+dover.Phase.RelPerm.GeomNames = "domain"
 
 dover.Geom.domain.RelPerm.Alpha = 6.0
-dover.Geom.domain.RelPerm.N = 2.
+dover.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.Phase.Saturation.Type = 'VanGenuchten'
-dover.Phase.Saturation.GeomNames = 'domain'
+dover.Phase.Saturation.Type = "VanGenuchten"
+dover.Phase.Saturation.GeomNames = "domain"
 
 dover.Geom.domain.Saturation.Alpha = 6.0
-dover.Geom.domain.Saturation.N = 2.
+dover.Geom.domain.Saturation.N = 2.0
 dover.Geom.domain.Saturation.SRes = 0.2
 dover.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Wells.Names = ''
+dover.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Cycle.Names = 'constant rainrec'
-dover.Cycle.constant.Names = 'alltime'
+dover.Cycle.Names = "constant rainrec"
+dover.Cycle.constant.Names = "alltime"
 dover.Cycle.constant.alltime.Length = 1
 dover.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-dover.Cycle.rainrec.Names = 'rain rec'
+dover.Cycle.rainrec.Names = "rain rec"
 dover.Cycle.rainrec.rain.Length = 1
 dover.Cycle.rainrec.rec.Length = 2
 dover.Cycle.rainrec.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dover.BCPressure.PatchNames = dover.Geom.domain.Patches
 
-dover.Patch.x_lower.BCPressure.Type = 'FluxConst'
-dover.Patch.x_lower.BCPressure.Cycle = 'constant'
+dover.Patch.x_lower.BCPressure.Type = "FluxConst"
+dover.Patch.x_lower.BCPressure.Cycle = "constant"
 dover.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-dover.Patch.y_lower.BCPressure.Type = 'FluxConst'
-dover.Patch.y_lower.BCPressure.Cycle = 'constant'
+dover.Patch.y_lower.BCPressure.Type = "FluxConst"
+dover.Patch.y_lower.BCPressure.Cycle = "constant"
 dover.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-dover.Patch.z_lower.BCPressure.Type = 'FluxConst'
-dover.Patch.z_lower.BCPressure.Cycle = 'constant'
+dover.Patch.z_lower.BCPressure.Type = "FluxConst"
+dover.Patch.z_lower.BCPressure.Cycle = "constant"
 dover.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-dover.Patch.x_upper.BCPressure.Type = 'FluxConst'
-dover.Patch.x_upper.BCPressure.Cycle = 'constant'
+dover.Patch.x_upper.BCPressure.Type = "FluxConst"
+dover.Patch.x_upper.BCPressure.Cycle = "constant"
 dover.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-dover.Patch.y_upper.BCPressure.Type = 'FluxConst'
-dover.Patch.y_upper.BCPressure.Cycle = 'constant'
+dover.Patch.y_upper.BCPressure.Type = "FluxConst"
+dover.Patch.y_upper.BCPressure.Cycle = "constant"
 dover.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
 ## overland flow boundary condition with very heavy rainfall then slight ET
-dover.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-dover.Patch.z_upper.BCPressure.Cycle = 'rainrec'
+dover.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+dover.Patch.z_upper.BCPressure.Cycle = "rainrec"
 dover.Patch.z_upper.BCPressure.rain.Value = -0.05
 dover.Patch.z_upper.BCPressure.rec.Value = 0.000001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.TopoSlopesX.Type = 'Constant'
-dover.TopoSlopesX.GeomNames = 'left right channel'
+dover.TopoSlopesX.Type = "Constant"
+dover.TopoSlopesX.GeomNames = "left right channel"
 dover.TopoSlopesX.Geom.left.Value = -0.005
 dover.TopoSlopesX.Geom.right.Value = 0.005
 dover.TopoSlopesX.Geom.channel.Value = 0.00
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.TopoSlopesY.Type = 'Constant'
-dover.TopoSlopesY.GeomNames = 'left right channel'
+dover.TopoSlopesY.Type = "Constant"
+dover.TopoSlopesY.GeomNames = "left right channel"
 dover.TopoSlopesY.Geom.left.Value = 0.001
 dover.TopoSlopesY.Geom.right.Value = 0.001
 dover.TopoSlopesY.Geom.channel.Value = 0.001
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-dover.Mannings.Type = 'Constant'
-dover.Mannings.GeomNames = 'left right channel'
-dover.Mannings.Geom.left.Value = 5.e-6
-dover.Mannings.Geom.right.Value = 5.e-6
-dover.Mannings.Geom.channel.Value = 1.e-6
+dover.Mannings.Type = "Constant"
+dover.Mannings.GeomNames = "left right channel"
+dover.Mannings.Geom.left.Value = 5.0e-6
+dover.Mannings.Geom.right.Value = 5.0e-6
+dover.Mannings.Geom.channel.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.PhaseSources.water.Type = 'Constant'
-dover.PhaseSources.water.GeomNames = 'domain'
+dover.PhaseSources.water.Type = "Constant"
+dover.PhaseSources.water.GeomNames = "domain"
 dover.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.KnownSolution = 'NoKnownSolution'
+dover.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Solver = 'Richards'
+dover.Solver = "Richards"
 dover.Solver.MaxIter = 2500
 
 dover.Solver.Nonlinear.MaxIter = 20
 dover.Solver.Nonlinear.ResidualTol = 1e-9
-dover.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+dover.Solver.Nonlinear.EtaChoice = "EtaConstant"
 dover.Solver.Nonlinear.EtaValue = 0.01
 dover.Solver.Nonlinear.UseJacobian = True
 dover.Solver.Nonlinear.DerivativeEpsilon = 1e-8
 dover.Solver.Nonlinear.StepTol = 1e-20
-dover.Solver.Nonlinear.Globalization = 'LineSearch'
+dover.Solver.Nonlinear.Globalization = "LineSearch"
 dover.Solver.Linear.KrylovDimension = 20
 dover.Solver.Linear.MaxRestart = 2
 
-dover.Solver.Linear.Preconditioner = 'PFMGOctree'
+dover.Solver.Linear.Preconditioner = "PFMGOctree"
 dover.Solver.PrintSubsurf = False
-dover.Solver.Drop = 1E-20
-dover.Solver.AbsTol = 1E-9
+dover.Solver.Drop = 1e-20
+dover.Solver.AbsTol = 1e-9
 
 dover.Solver.WriteSiloSubsurfData = True
 dover.Solver.WriteSiloPressure = True
 dover.Solver.WriteSiloSaturation = True
 dover.Solver.WriteSiloConcentration = True
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be at the bottom of the domain, the top layer is initially dry
-dover.ICPressure.Type = 'HydroStaticPatch'
-dover.ICPressure.GeomNames = 'domain'
+dover.ICPressure.Type = "HydroStaticPatch"
+dover.ICPressure.GeomNames = "domain"
 dover.Geom.domain.ICPressure.Value = -3.0
 
-dover.Geom.domain.ICPressure.RefGeom = 'domain'
-dover.Geom.domain.ICPressure.RefPatch = 'z_upper'
+dover.Geom.domain.ICPressure.RefGeom = "domain"
+dover.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/dover_poj')
+dir_name = get_absolute_path("test_output/dover_poj")
 mkdir(dir_name)
 dover.run(working_directory=dir_name)
diff --git a/test/python/default_overland.py b/test/python/default_overland.py
index 5a1aa0405..2df3dda83 100644
--- a/test/python/default_overland.py
+++ b/test/python/default_overland.py
@@ -1,24 +1,33 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  This runs the tilted-v catchment problem
 #  similar to that in Kollet and Maxwell (2006) AWR
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
+import sys, argparse
 from parflow import Run
-from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.fs import mkdir, get_absolute_path, rm
+from parflow.tools.compare import pf_test_file, pf_test_file_with_abs
 
-dover = Run("dover", __file__)
+run_name = "default_overland"
+dover = Run(run_name, __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 dover.FileVersion = 4
 
-dover.Process.Topology.P = 1
-dover.Process.Topology.Q = 1
-dover.Process.Topology.R = 1
+parser = argparse.ArgumentParser()
+parser.add_argument("-p", "--p", default=1)
+parser.add_argument("-q", "--q", default=1)
+parser.add_argument("-r", "--r", default=1)
+args = parser.parse_args()
 
-#---------------------------------------------------------
+dover.Process.Topology.P = args.p
+dover.Process.Topology.Q = args.q
+dover.Process.Topology.R = args.r
+
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 dover.ComputationalGrid.Lower.X = 0.0
 dover.ComputationalGrid.Lower.Y = 0.0
@@ -30,27 +39,27 @@
 
 dover.ComputationalGrid.DX = 10.0
 dover.ComputationalGrid.DY = 10.0
-dover.ComputationalGrid.DZ = .05
+dover.ComputationalGrid.DZ = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.GeomInput.Names = 'domaininput leftinput rightinput channelinput'
+dover.GeomInput.Names = "domaininput leftinput rightinput channelinput"
 
-dover.GeomInput.domaininput.GeomName = 'domain'
-dover.GeomInput.leftinput.GeomName = 'left'
-dover.GeomInput.rightinput.GeomName = 'right'
-dover.GeomInput.channelinput.GeomName = 'channel'
+dover.GeomInput.domaininput.GeomName = "domain"
+dover.GeomInput.leftinput.GeomName = "left"
+dover.GeomInput.rightinput.GeomName = "right"
+dover.GeomInput.channelinput.GeomName = "channel"
 
-dover.GeomInput.domaininput.InputType = 'Box'
-dover.GeomInput.leftinput.InputType = 'Box'
-dover.GeomInput.rightinput.InputType = 'Box'
-dover.GeomInput.channelinput.InputType = 'Box'
+dover.GeomInput.domaininput.InputType = "Box"
+dover.GeomInput.leftinput.InputType = "Box"
+dover.GeomInput.rightinput.InputType = "Box"
+dover.GeomInput.channelinput.InputType = "Box"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 dover.Geom.domain.Lower.X = 0.0
 dover.Geom.domain.Lower.Y = 0.0
@@ -59,11 +68,11 @@
 dover.Geom.domain.Upper.X = 300.0
 dover.Geom.domain.Upper.Y = 300.0
 dover.Geom.domain.Upper.Z = 1.5
-dover.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+dover.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Left Slope Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 dover.Geom.left.Lower.X = 0.0
 dover.Geom.left.Lower.Y = 0.0
@@ -73,9 +82,9 @@
 dover.Geom.left.Upper.Y = 300.0
 dover.Geom.left.Upper.Z = 1.5
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Right Slope Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 dover.Geom.right.Lower.X = 160.0
 dover.Geom.right.Lower.Y = 0.0
@@ -85,9 +94,9 @@
 dover.Geom.right.Upper.Y = 300.0
 dover.Geom.right.Upper.Z = 1.5
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Channel Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 dover.Geom.channel.Lower.X = 140.0
 dover.Geom.channel.Lower.Y = 0.0
@@ -97,11 +106,11 @@
 dover.Geom.channel.Upper.Y = 300.0
 dover.Geom.channel.Upper.Z = 1.5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Geom.Perm.Names = 'left right channel'
+dover.Geom.Perm.Names = "left right channel"
 
 # Values in m/hour
 
@@ -109,9 +118,9 @@
 # the following is ignored if the perm.type "Constant" settings are not
 # commented out, below.
 
-dover.Geom.left.Perm.Type = 'TurnBands'
-dover.Geom.left.Perm.LambdaX = 50.
-dover.Geom.left.Perm.LambdaY = 50.
+dover.Geom.left.Perm.Type = "TurnBands"
+dover.Geom.left.Perm.LambdaX = 50.0
+dover.Geom.left.Perm.LambdaY = 50.0
 dover.Geom.left.Perm.LambdaZ = 0.5
 dover.Geom.left.Perm.GeomMean = 0.01
 
@@ -121,12 +130,12 @@
 dover.Geom.left.Perm.KMax = 100.0
 dover.Geom.left.Perm.DelK = 0.2
 dover.Geom.left.Perm.Seed = 33333
-dover.Geom.left.Perm.LogNormal = 'Log'
-dover.Geom.left.Perm.StratType = 'Bottom'
+dover.Geom.left.Perm.LogNormal = "Log"
+dover.Geom.left.Perm.StratType = "Bottom"
 
-dover.Geom.right.Perm.Type = 'TurnBands'
-dover.Geom.right.Perm.LambdaX = 50.
-dover.Geom.right.Perm.LambdaY = 50.
+dover.Geom.right.Perm.Type = "TurnBands"
+dover.Geom.right.Perm.LambdaX = 50.0
+dover.Geom.right.Perm.LambdaY = 50.0
 dover.Geom.right.Perm.LambdaZ = 0.5
 dover.Geom.right.Perm.GeomMean = 0.05
 
@@ -136,8 +145,8 @@
 dover.Geom.right.Perm.KMax = 100.0
 dover.Geom.right.Perm.DelK = 0.2
 dover.Geom.right.Perm.Seed = 13333
-dover.Geom.right.Perm.LogNormal = 'Log'
-dover.Geom.right.Perm.StratType = 'Bottom'
+dover.Geom.right.Perm.LogNormal = "Log"
+dover.Geom.right.Perm.StratType = "Bottom"
 
 # hydraulic conductivity is very low, but not zero, top node will have to saturate
 # before overland flow can begin and will be driven by hortonian flow
@@ -145,254 +154,308 @@
 # turning bands above.  Run time increases quite a bit with a heterogeneous
 # subsurface
 
-dover.Geom.left.Perm.Type = 'Constant'
+dover.Geom.left.Perm.Type = "Constant"
 dover.Geom.left.Perm.Value = 0.001
 
-dover.Geom.right.Perm.Type = 'Constant'
+dover.Geom.right.Perm.Type = "Constant"
 dover.Geom.right.Perm.Value = 0.01
 
-dover.Geom.channel.Perm.Type = 'Constant'
+dover.Geom.channel.Perm.Type = "Constant"
 dover.Geom.channel.Perm.Value = 0.00001
 
-dover.Perm.TensorType = 'TensorByGeom'
+dover.Perm.TensorType = "TensorByGeom"
 
-dover.Geom.Perm.TensorByGeom.Names = 'domain'
+dover.Geom.Perm.TensorByGeom.Names = "domain"
 
 dover.Geom.domain.Perm.TensorValX = 1.0
 dover.Geom.domain.Perm.TensorValY = 1.0
 dover.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.SpecificStorage.Type = 'Constant'
-dover.SpecificStorage.GeomNames = 'domain'
+dover.SpecificStorage.Type = "Constant"
+dover.SpecificStorage.GeomNames = "domain"
 dover.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Phase.Names = 'water'
+dover.Phase.Names = "water"
 
-dover.Phase.water.Density.Type = 'Constant'
+dover.Phase.water.Density.Type = "Constant"
 dover.Phase.water.Density.Value = 1.0
 
-dover.Phase.water.Viscosity.Type = 'Constant'
+dover.Phase.water.Viscosity.Type = "Constant"
 dover.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Contaminants.Names = ''
+dover.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Geom.Retardation.GeomNames = ''
+dover.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dover.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dover.TimingInfo.BaseUnit = 0.1
 dover.TimingInfo.StartCount = 0
 dover.TimingInfo.StartTime = 0.0
 dover.TimingInfo.StopTime = 0.4
 dover.TimingInfo.DumpInterval = -1
-dover.TimeStep.Type = 'Constant'
+dover.TimeStep.Type = "Constant"
 dover.TimeStep.Value = 0.1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Geom.Porosity.GeomNames = 'left right channel'
+dover.Geom.Porosity.GeomNames = "left right channel"
 
-dover.Geom.left.Porosity.Type = 'Constant'
+dover.Geom.left.Porosity.Type = "Constant"
 dover.Geom.left.Porosity.Value = 0.25
 
-dover.Geom.right.Porosity.Type = 'Constant'
+dover.Geom.right.Porosity.Type = "Constant"
 dover.Geom.right.Porosity.Value = 0.25
 
-dover.Geom.channel.Porosity.Type = 'Constant'
+dover.Geom.channel.Porosity.Type = "Constant"
 dover.Geom.channel.Porosity.Value = 0.01
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Domain.GeomName = 'domain'
+dover.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Phase.RelPerm.Type = 'VanGenuchten'
-dover.Phase.RelPerm.GeomNames = 'domain'
+dover.Phase.RelPerm.Type = "VanGenuchten"
+dover.Phase.RelPerm.GeomNames = "domain"
 
 dover.Geom.domain.RelPerm.Alpha = 6.0
-dover.Geom.domain.RelPerm.N = 2.
+dover.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.Phase.Saturation.Type = 'VanGenuchten'
-dover.Phase.Saturation.GeomNames = 'domain'
+dover.Phase.Saturation.Type = "VanGenuchten"
+dover.Phase.Saturation.GeomNames = "domain"
 
 dover.Geom.domain.Saturation.Alpha = 6.0
-dover.Geom.domain.Saturation.N = 2.
+dover.Geom.domain.Saturation.N = 2.0
 dover.Geom.domain.Saturation.SRes = 0.2
 dover.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Wells.Names = ''
+dover.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Cycle.Names = 'constant rainrec'
-dover.Cycle.constant.Names = 'alltime'
+dover.Cycle.Names = "constant rainrec"
+dover.Cycle.constant.Names = "alltime"
 dover.Cycle.constant.alltime.Length = 1
 dover.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-dover.Cycle.rainrec.Names = 'rain rec'
+dover.Cycle.rainrec.Names = "rain rec"
 dover.Cycle.rainrec.rain.Length = 1
 dover.Cycle.rainrec.rec.Length = 2
 dover.Cycle.rainrec.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dover.BCPressure.PatchNames = dover.Geom.domain.Patches
 
-dover.Patch.x_lower.BCPressure.Type = 'FluxConst'
-dover.Patch.x_lower.BCPressure.Cycle = 'constant'
+dover.Patch.x_lower.BCPressure.Type = "FluxConst"
+dover.Patch.x_lower.BCPressure.Cycle = "constant"
 dover.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-dover.Patch.y_lower.BCPressure.Type = 'FluxConst'
-dover.Patch.y_lower.BCPressure.Cycle = 'constant'
+dover.Patch.y_lower.BCPressure.Type = "FluxConst"
+dover.Patch.y_lower.BCPressure.Cycle = "constant"
 dover.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-dover.Patch.z_lower.BCPressure.Type = 'FluxConst'
-dover.Patch.z_lower.BCPressure.Cycle = 'constant'
+dover.Patch.z_lower.BCPressure.Type = "FluxConst"
+dover.Patch.z_lower.BCPressure.Cycle = "constant"
 dover.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-dover.Patch.x_upper.BCPressure.Type = 'FluxConst'
-dover.Patch.x_upper.BCPressure.Cycle = 'constant'
+dover.Patch.x_upper.BCPressure.Type = "FluxConst"
+dover.Patch.x_upper.BCPressure.Cycle = "constant"
 dover.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-dover.Patch.y_upper.BCPressure.Type = 'FluxConst'
-dover.Patch.y_upper.BCPressure.Cycle = 'constant'
+dover.Patch.y_upper.BCPressure.Type = "FluxConst"
+dover.Patch.y_upper.BCPressure.Cycle = "constant"
 dover.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
 ## overland flow boundary condition with very heavy rainfall then slight ET
-dover.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-dover.Patch.z_upper.BCPressure.Cycle = 'rainrec'
+dover.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+dover.Patch.z_upper.BCPressure.Cycle = "rainrec"
 dover.Patch.z_upper.BCPressure.rain.Value = -0.05
 dover.Patch.z_upper.BCPressure.rec.Value = 0.000001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.TopoSlopesX.Type = 'Constant'
-dover.TopoSlopesX.GeomNames = 'left right channel'
+dover.TopoSlopesX.Type = "Constant"
+dover.TopoSlopesX.GeomNames = "left right channel"
 dover.TopoSlopesX.Geom.left.Value = -0.005
 dover.TopoSlopesX.Geom.right.Value = 0.005
 dover.TopoSlopesX.Geom.channel.Value = 0.00
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.TopoSlopesY.Type = 'Constant'
-dover.TopoSlopesY.GeomNames = 'left right channel'
+dover.TopoSlopesY.Type = "Constant"
+dover.TopoSlopesY.GeomNames = "left right channel"
 dover.TopoSlopesY.Geom.left.Value = 0.001
 dover.TopoSlopesY.Geom.right.Value = 0.001
 dover.TopoSlopesY.Geom.channel.Value = 0.001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.Mannings.Type = 'Constant'
-dover.Mannings.GeomNames = 'left right channel'
-dover.Mannings.Geom.left.Value = 5.e-6
-dover.Mannings.Geom.right.Value = 5.e-6
-dover.Mannings.Geom.channel.Value = 1.e-6
+dover.Mannings.Type = "Constant"
+dover.Mannings.GeomNames = "left right channel"
+dover.Mannings.Geom.left.Value = 5.0e-6
+dover.Mannings.Geom.right.Value = 5.0e-6
+dover.Mannings.Geom.channel.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.PhaseSources.water.Type = 'Constant'
-dover.PhaseSources.water.GeomNames = 'domain'
+dover.PhaseSources.water.Type = "Constant"
+dover.PhaseSources.water.GeomNames = "domain"
 dover.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.KnownSolution = 'NoKnownSolution'
+dover.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Solver = 'Richards'
+dover.Solver = "Richards"
 dover.Solver.MaxIter = 2500
 
 dover.Solver.Nonlinear.MaxIter = 20
 dover.Solver.Nonlinear.ResidualTol = 1e-9
-dover.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+dover.Solver.Nonlinear.EtaChoice = "EtaConstant"
 dover.Solver.Nonlinear.EtaValue = 0.01
 dover.Solver.Nonlinear.UseJacobian = False
 dover.Solver.Nonlinear.DerivativeEpsilon = 1e-8
 dover.Solver.Nonlinear.StepTol = 1e-20
-dover.Solver.Nonlinear.Globalization = 'LineSearch'
+dover.Solver.Nonlinear.Globalization = "LineSearch"
 dover.Solver.Linear.KrylovDimension = 20
 dover.Solver.Linear.MaxRestart = 2
 
-dover.Solver.Linear.Preconditioner = 'PFMGOctree'
+dover.Solver.Linear.Preconditioner = "PFMGOctree"
 dover.Solver.PrintSubsurf = False
-dover.Solver.Drop = 1E-20
-dover.Solver.AbsTol = 1E-9
+dover.Solver.Drop = 1e-20
+dover.Solver.AbsTol = 1e-9
+
+dover.Solver.PrintVelocities = True
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be at the bottom of the domain, the top layer is initially dry
-dover.ICPressure.Type = 'HydroStaticPatch'
-dover.ICPressure.GeomNames = 'domain'
+dover.ICPressure.Type = "HydroStaticPatch"
+dover.ICPressure.GeomNames = "domain"
 dover.Geom.domain.ICPressure.Value = -3.0
 
-dover.Geom.domain.ICPressure.RefGeom = 'domain'
-dover.Geom.domain.ICPressure.RefPatch = 'z_upper'
+dover.Geom.domain.ICPressure.RefGeom = "domain"
+dover.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/dover')
-mkdir(dir_name)
-dover.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/default_overland")
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+dover.run(working_directory=new_output_dir_name)
+
+passed = True
+sig_digits = 5
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+        sig_digits,
+    ):
+        passed = False
+
+test_files = ["press", "satur"]
+for i in range(5):
+    for test_file in test_files:
+        timestep = str(i).rjust(5, "0")
+        filename = f"/{run_name}.out.{test_file}.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in {test_file} for timestep {timestep}",
+            sig_digits,
+        ):
+            passed = False
+
+
+abs_value = 1e-12
+test_files = ["velx", "vely", "velz"]
+for i in range(5):
+    for test_file in test_files:
+        timestep = str(i).rjust(5, "0")
+        filename = f"/{run_name}.out.{test_file}.{timestep}.pfb"
+        if not pf_test_file_with_abs(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in {test_file} for timestep {timestep}",
+            abs_value,
+            sig_digits,
+        ):
+            passed = False
+
+
+rm(new_output_dir_name)
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/default_richards.py b/test/python/default_richards.py
index e5f665d05..c26dc70d6 100644
--- a/test/python/default_richards.py
+++ b/test/python/default_richards.py
@@ -1,340 +1,419 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  This runs the basic default_richards test case.
 #  This run, as written in this input file, should take
 #  3 nonlinear iterations.
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
+import sys, argparse
 from parflow import Run
-from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.fs import mkdir, get_absolute_path, rm
+from parflow.tools.compare import pf_test_file, pf_test_file_with_abs
 
-drich = Run('default_richards', __file__)
+run_name = "default_richards"
 
-#---------------------------------------------------------
+default_richards = Run(run_name, __file__)
 
-drich.Process.Topology.P = 1
-drich.Process.Topology.Q = 1
-drich.Process.Topology.R = 1
+# ---------------------------------------------------------
+parser = argparse.ArgumentParser()
+parser.add_argument("-p", "--p", default=1)
+parser.add_argument("-q", "--q", default=1)
+parser.add_argument("-r", "--r", default=1)
+args = parser.parse_args()
 
-#---------------------------------------------------------
+default_richards.Process.Topology.P = args.p
+default_richards.Process.Topology.Q = args.q
+default_richards.Process.Topology.R = args.r
+
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.ComputationalGrid.Lower.X = -10.0
-drich.ComputationalGrid.Lower.Y = 10.0
-drich.ComputationalGrid.Lower.Z = 1.0
+default_richards.ComputationalGrid.Lower.X = -10.0
+default_richards.ComputationalGrid.Lower.Y = 10.0
+default_richards.ComputationalGrid.Lower.Z = 1.0
 
-drich.ComputationalGrid.DX = 8.8888888888888893
-drich.ComputationalGrid.DY = 10.666666666666666
-drich.ComputationalGrid.DZ = 1.0
+default_richards.ComputationalGrid.DX = 8.8888888888888893
+default_richards.ComputationalGrid.DY = 10.666666666666666
+default_richards.ComputationalGrid.DZ = 1.0
 
-drich.ComputationalGrid.NX = 10
-drich.ComputationalGrid.NY = 10
-drich.ComputationalGrid.NZ = 8
+default_richards.ComputationalGrid.NX = 18
+default_richards.ComputationalGrid.NY = 15
+default_richards.ComputationalGrid.NZ = 8
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.Names = "domain_input background_input source_region_input concen_region_input"
+default_richards.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.domain_input.InputType = 'Box'
-drich.GeomInput.domain_input.GeomName = 'domain'
+default_richards.GeomInput.domain_input.InputType = "Box"
+default_richards.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.Geom.domain.Lower.X = -10.0
-drich.Geom.domain.Lower.Y = 10.0
-drich.Geom.domain.Lower.Z = 1.0
+default_richards.Geom.domain.Lower.X = -10.0
+default_richards.Geom.domain.Lower.Y = 10.0
+default_richards.Geom.domain.Lower.Z = 1.0
 
-drich.Geom.domain.Upper.X = 150.0
-drich.Geom.domain.Upper.Y = 170.0
-drich.Geom.domain.Upper.Z = 9.0
+default_richards.Geom.domain.Upper.X = 150.0
+default_richards.Geom.domain.Upper.Y = 170.0
+default_richards.Geom.domain.Upper.Z = 9.0
 
-drich.Geom.domain.Patches = "left right front back bottom top"
+default_richards.Geom.domain.Patches = "left right front back bottom top"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.background_input.InputType = 'Box'
-drich.GeomInput.background_input.GeomName = 'background'
+default_richards.GeomInput.background_input.InputType = "Box"
+default_richards.GeomInput.background_input.GeomName = "background"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.Geom.background.Lower.X = -99999999.0
-drich.Geom.background.Lower.Y = -99999999.0
-drich.Geom.background.Lower.Z = -99999999.0
+default_richards.Geom.background.Lower.X = -99999999.0
+default_richards.Geom.background.Lower.Y = -99999999.0
+default_richards.Geom.background.Lower.Z = -99999999.0
 
-drich.Geom.background.Upper.X = 99999999.0
-drich.Geom.background.Upper.Y = 99999999.0
-drich.Geom.background.Upper.Z = 99999999.0
+default_richards.Geom.background.Upper.X = 99999999.0
+default_richards.Geom.background.Upper.Y = 99999999.0
+default_richards.Geom.background.Upper.Z = 99999999.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.source_region_input.InputType = 'Box'
-drich.GeomInput.source_region_input.GeomName = 'source_region'
+default_richards.GeomInput.source_region_input.InputType = "Box"
+default_richards.GeomInput.source_region_input.GeomName = "source_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.Geom.source_region.Lower.X = 65.56
-drich.Geom.source_region.Lower.Y = 79.34
-drich.Geom.source_region.Lower.Z = 4.5
+default_richards.Geom.source_region.Lower.X = 65.56
+default_richards.Geom.source_region.Lower.Y = 79.34
+default_richards.Geom.source_region.Lower.Z = 4.5
 
-drich.Geom.source_region.Upper.X = 74.44
-drich.Geom.source_region.Upper.Y = 89.99
-drich.Geom.source_region.Upper.Z = 5.5
+default_richards.Geom.source_region.Upper.X = 74.44
+default_richards.Geom.source_region.Upper.Y = 89.99
+default_richards.Geom.source_region.Upper.Z = 5.5
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.concen_region_input.InputType = 'Box'
-drich.GeomInput.concen_region_input.GeomName = 'concen_region'
+default_richards.GeomInput.concen_region_input.InputType = "Box"
+default_richards.GeomInput.concen_region_input.GeomName = "concen_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.Geom.concen_region.Lower.X = 60.0
-drich.Geom.concen_region.Lower.Y = 80.0
-drich.Geom.concen_region.Lower.Z = 4.0
+default_richards.Geom.concen_region.Lower.X = 60.0
+default_richards.Geom.concen_region.Lower.Y = 80.0
+default_richards.Geom.concen_region.Lower.Z = 4.0
 
-drich.Geom.concen_region.Upper.X = 80.0
-drich.Geom.concen_region.Upper.Y = 100.0
-drich.Geom.concen_region.Upper.Z = 6.0
+default_richards.Geom.concen_region.Upper.X = 80.0
+default_richards.Geom.concen_region.Upper.Y = 100.0
+default_richards.Geom.concen_region.Upper.Z = 6.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Geom.Perm.Names = "background"
+default_richards.Geom.Perm.Names = "background"
 
-drich.Geom.background.Perm.Type = 'Constant'
-drich.Geom.background.Perm.Value = 4.0
+default_richards.Geom.background.Perm.Type = "Constant"
+default_richards.Geom.background.Perm.Value = 4.0
 
-drich.Perm.TensorType = 'TensorByGeom'
+default_richards.Perm.TensorType = "TensorByGeom"
 
-drich.Geom.Perm.TensorByGeom.Names = "background"
+default_richards.Geom.Perm.TensorByGeom.Names = "background"
 
-drich.Geom.background.Perm.TensorValX = 1.0
-drich.Geom.background.Perm.TensorValY = 1.0
-drich.Geom.background.Perm.TensorValZ = 1.0
+default_richards.Geom.background.Perm.TensorValX = 1.0
+default_richards.Geom.background.Perm.TensorValY = 1.0
+default_richards.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.SpecificStorage.Type = 'Constant'
-drich.SpecificStorage.GeomNames = "domain"
-drich.Geom.domain.SpecificStorage.Value = 1.0e-4
+default_richards.SpecificStorage.Type = "Constant"
+default_richards.SpecificStorage.GeomNames = "domain"
+default_richards.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Phase.Names = "water"
+default_richards.Phase.Names = "water"
 
-drich.Phase.water.Density.Type = 'Constant'
-drich.Phase.water.Density.Value = 1.0
+default_richards.Phase.water.Density.Type = "Constant"
+default_richards.Phase.water.Density.Value = 1.0
 
-drich.Phase.water.Viscosity.Type = 'Constant'
-drich.Phase.water.Viscosity.Value = 1.0
+default_richards.Phase.water.Viscosity.Type = "Constant"
+default_richards.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Contaminants.Names = ""
+default_richards.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Geom.Retardation.GeomNames = ""
+default_richards.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Gravity = 1.0
+default_richards.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.TimingInfo.BaseUnit = 1.0
-drich.TimingInfo.StartCount = 0
-drich.TimingInfo.StartTime = 0.0
-drich.TimingInfo.StopTime = 0.010
-drich.TimingInfo.DumpInterval = -1
-drich.TimeStep.Type = 'Constant'
-drich.TimeStep.Value = 0.001
+default_richards.TimingInfo.BaseUnit = 1.0
+default_richards.TimingInfo.StartCount = 0
+default_richards.TimingInfo.StartTime = 0.0
+default_richards.TimingInfo.StopTime = 1.0
+default_richards.TimingInfo.DumpInterval = -1
+default_richards.TimeStep.Type = "Constant"
+default_richards.TimeStep.Value = 0.001
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Geom.Porosity.GeomNames = 'background'
-drich.Geom.background.Porosity.Type = 'Constant'
-drich.Geom.background.Porosity.Value = 1.0
+default_richards.Geom.Porosity.GeomNames = "background"
+default_richards.Geom.background.Porosity.Type = "Constant"
+default_richards.Geom.background.Porosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Domain.GeomName = 'domain'
+default_richards.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Phase.RelPerm.Type = 'VanGenuchten'
-drich.Phase.RelPerm.GeomNames = 'domain'
-drich.Geom.domain.RelPerm.Alpha = 0.005
-drich.Geom.domain.RelPerm.N = 2.0
+default_richards.Phase.RelPerm.Type = "VanGenuchten"
+default_richards.Phase.RelPerm.GeomNames = "domain"
+default_richards.Geom.domain.RelPerm.Alpha = 0.005
+default_richards.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.Phase.Saturation.Type = 'VanGenuchten'
-drich.Phase.Saturation.GeomNames = 'domain'
-drich.Geom.domain.Saturation.Alpha = 0.005
-drich.Geom.domain.Saturation.N = 2.0
-drich.Geom.domain.Saturation.SRes = 0.2
-drich.Geom.domain.Saturation.SSat = 0.99
+default_richards.Phase.Saturation.Type = "VanGenuchten"
+default_richards.Phase.Saturation.GeomNames = "domain"
+default_richards.Geom.domain.Saturation.Alpha = 0.005
+default_richards.Geom.domain.Saturation.N = 2.0
+default_richards.Geom.domain.Saturation.SRes = 0.2
+default_richards.Geom.domain.Saturation.SSat = 0.99
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Wells.Names = ''
+default_richards.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Cycle.Names = 'constant'
-drich.Cycle.constant.Names = "alltime"
-drich.Cycle.constant.alltime.Length = 1
-drich.Cycle.constant.Repeat = -1
+default_richards.Cycle.Names = "constant"
+default_richards.Cycle.constant.Names = "alltime"
+default_richards.Cycle.constant.alltime.Length = 1
+default_richards.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.BCPressure.PatchNames = "left right front back bottom top"
+default_richards.BCPressure.PatchNames = "left right front back bottom top"
 
-drich.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-drich.Patch.left.BCPressure.Cycle = "constant"
-drich.Patch.left.BCPressure.RefGeom = 'domain'
-drich.Patch.left.BCPressure.RefPatch = 'bottom'
-drich.Patch.left.BCPressure.alltime.Value = 5.0
+default_richards.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+default_richards.Patch.left.BCPressure.Cycle = "constant"
+default_richards.Patch.left.BCPressure.RefGeom = "domain"
+default_richards.Patch.left.BCPressure.RefPatch = "bottom"
+default_richards.Patch.left.BCPressure.alltime.Value = 5.0
 
-drich.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-drich.Patch.right.BCPressure.Cycle = "constant"
-drich.Patch.right.BCPressure.RefGeom = 'domain'
-drich.Patch.right.BCPressure.RefPatch = 'bottom'
-drich.Patch.right.BCPressure.alltime.Value = 3.0
+default_richards.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+default_richards.Patch.right.BCPressure.Cycle = "constant"
+default_richards.Patch.right.BCPressure.RefGeom = "domain"
+default_richards.Patch.right.BCPressure.RefPatch = "bottom"
+default_richards.Patch.right.BCPressure.alltime.Value = 3.0
 
-drich.Patch.front.BCPressure.Type = 'FluxConst'
-drich.Patch.front.BCPressure.Cycle = "constant"
-drich.Patch.front.BCPressure.alltime.Value = 0.0
+default_richards.Patch.front.BCPressure.Type = "FluxConst"
+default_richards.Patch.front.BCPressure.Cycle = "constant"
+default_richards.Patch.front.BCPressure.alltime.Value = 0.0
 
-drich.Patch.back.BCPressure.Type = 'FluxConst'
-drich.Patch.back.BCPressure.Cycle = "constant"
-drich.Patch.back.BCPressure.alltime.Value = 0.0
+default_richards.Patch.back.BCPressure.Type = "FluxConst"
+default_richards.Patch.back.BCPressure.Cycle = "constant"
+default_richards.Patch.back.BCPressure.alltime.Value = 0.0
 
-drich.Patch.bottom.BCPressure.Type = 'FluxConst'
-drich.Patch.bottom.BCPressure.Cycle = "constant"
-drich.Patch.bottom.BCPressure.alltime.Value = 0.0
+default_richards.Patch.bottom.BCPressure.Type = "FluxConst"
+default_richards.Patch.bottom.BCPressure.Cycle = "constant"
+default_richards.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-drich.Patch.top.BCPressure.Type = 'FluxConst'
-drich.Patch.top.BCPressure.Cycle = "constant"
-drich.Patch.top.BCPressure.alltime.Value = 0.0
+default_richards.Patch.top.BCPressure.Type = "FluxConst"
+default_richards.Patch.top.BCPressure.Cycle = "constant"
+default_richards.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.TopoSlopesX.Type = "Constant"
-drich.TopoSlopesX.GeomNames = "domain"
-drich.TopoSlopesX.Geom.domain.Value = 0.0
+default_richards.TopoSlopesX.Type = "Constant"
+default_richards.TopoSlopesX.GeomNames = "domain"
+default_richards.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.TopoSlopesY.Type = "Constant"
-drich.TopoSlopesY.GeomNames = "domain"
-drich.TopoSlopesY.Geom.domain.Value = 0.0
+default_richards.TopoSlopesY.Type = "Constant"
+default_richards.TopoSlopesY.GeomNames = "domain"
+default_richards.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.Mannings.Type = "Constant"
-drich.Mannings.GeomNames = "domain"
-drich.Mannings.Geom.domain.Value = 0.
+default_richards.Mannings.Type = "Constant"
+default_richards.Mannings.GeomNames = "domain"
+default_richards.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.ICPressure.Type = 'HydroStaticPatch'
-drich.ICPressure.GeomNames = 'domain'
-drich.Geom.domain.ICPressure.Value = 3.0
-drich.Geom.domain.ICPressure.RefGeom = 'domain'
-drich.Geom.domain.ICPressure.RefPatch = 'bottom'
+default_richards.ICPressure.Type = "HydroStaticPatch"
+default_richards.ICPressure.GeomNames = "domain"
+default_richards.Geom.domain.ICPressure.Value = 3.0
+default_richards.Geom.domain.ICPressure.RefGeom = "domain"
+default_richards.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.PhaseSources.water.Type = 'Constant'
-drich.PhaseSources.water.GeomNames = 'background'
-drich.PhaseSources.water.Geom.background.Value = 0.0
+default_richards.PhaseSources.water.Type = "Constant"
+default_richards.PhaseSources.water.GeomNames = "background"
+default_richards.PhaseSources.water.Geom.background.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.KnownSolution = 'NoKnownSolution'
+default_richards.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Solver = 'Richards'
-drich.Solver.MaxIter = 5
+default_richards.Solver = "Richards"
+default_richards.Solver.MaxIter = 50
 
-drich.Solver.Nonlinear.MaxIter = 10
-drich.Solver.Nonlinear.ResidualTol = 1e-9
-drich.Solver.Nonlinear.EtaChoice = 'EtaConstant'
-drich.Solver.Nonlinear.EtaValue = 1e-5
-drich.Solver.Nonlinear.UseJacobian = True
-drich.Solver.Nonlinear.DerivativeEpsilon = 1e-2
+default_richards.Solver.Nonlinear.MaxIter = 10
+default_richards.Solver.Nonlinear.ResidualTol = 1e-9
+default_richards.Solver.Nonlinear.EtaChoice = "EtaConstant"
+default_richards.Solver.Nonlinear.EtaValue = 1e-5
+default_richards.Solver.Nonlinear.UseJacobian = True
+default_richards.Solver.Nonlinear.DerivativeEpsilon = 1e-2
 
-drich.Solver.Linear.KrylovDimension = 10
+default_richards.Solver.Linear.KrylovDimension = 10
 
-drich.Solver.Linear.Preconditioner = 'PFMG'
+default_richards.Solver.Linear.Preconditioner = "PFMG"
 
-#-----------------------------------------------------------------------------
-# Run ParFlow
-#-----------------------------------------------------------------------------
+default_richards.Solver.PrintVelocities = True
 
-dir_name = get_absolute_path('test_output/drich')
-mkdir(dir_name)
-drich.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+# Run ParFlow
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/default_richards")
+mkdir(new_output_dir_name)
+correct_output_dir_name = get_absolute_path("../correct_output")
+default_richards.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+
+sig_digits = 6
+abs_value = 1e-12
+for i in range(51):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.velx.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in x-velocity for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+    filename = f"/{run_name}.out.vely.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in y-velocity for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+    filename = f"/{run_name}.out.vely.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in z-velocity for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/default_richards_nocluster.py b/test/python/default_richards_nocluster.py
index 50a6200c9..633aea5ff 100644
--- a/test/python/default_richards_nocluster.py
+++ b/test/python/default_richards_nocluster.py
@@ -1,15 +1,18 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 #  This runs the basic default_richards test case.
 #  This run, as written in this input file, should take
 #  3 nonlinear iterations.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
-drich_n = Run("default_richards_nocluster", __file__)
+run_name = "default_richards"
+drich_n = Run(run_name, __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich_n.FileVersion = 4
 
@@ -19,9 +22,9 @@
 drich_n.Process.Topology.Q = 1
 drich_n.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich_n.ComputationalGrid.Lower.X = -10.0
 drich_n.ComputationalGrid.Lower.Y = 10.0
@@ -31,26 +34,28 @@
 drich_n.ComputationalGrid.DY = 10.666666666666666
 drich_n.ComputationalGrid.DZ = 1.0
 
-drich_n.ComputationalGrid.NX = 10
-drich_n.ComputationalGrid.NY = 10
+drich_n.ComputationalGrid.NX = 18
+drich_n.ComputationalGrid.NY = 15
 drich_n.ComputationalGrid.NZ = 8
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich_n.GeomInput.Names = 'domain_input background_input source_region_input concen_region_input'
+drich_n.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich_n.GeomInput.domain_input.InputType = 'Box'
-drich_n.GeomInput.domain_input.GeomName = 'domain'
+drich_n.GeomInput.domain_input.InputType = "Box"
+drich_n.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich_n.Geom.domain.Lower.X = -10.0
 drich_n.Geom.domain.Lower.Y = 10.0
@@ -60,18 +65,18 @@
 drich_n.Geom.domain.Upper.Y = 170.0
 drich_n.Geom.domain.Upper.Z = 9.0
 
-drich_n.Geom.domain.Patches = 'left right front back bottom top'
+drich_n.Geom.domain.Patches = "left right front back bottom top"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich_n.GeomInput.background_input.InputType = 'Box'
-drich_n.GeomInput.background_input.GeomName = 'background'
+drich_n.GeomInput.background_input.InputType = "Box"
+drich_n.GeomInput.background_input.GeomName = "background"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich_n.Geom.background.Lower.X = -99999999.0
 drich_n.Geom.background.Lower.Y = -99999999.0
@@ -81,16 +86,16 @@
 drich_n.Geom.background.Upper.Y = 99999999.0
 drich_n.Geom.background.Upper.Z = 99999999.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich_n.GeomInput.source_region_input.InputType = 'Box'
-drich_n.GeomInput.source_region_input.GeomName = 'source_region'
+drich_n.GeomInput.source_region_input.InputType = "Box"
+drich_n.GeomInput.source_region_input.GeomName = "source_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich_n.Geom.source_region.Lower.X = 65.56
 drich_n.Geom.source_region.Lower.Y = 79.34
@@ -100,16 +105,16 @@
 drich_n.Geom.source_region.Upper.Y = 89.99
 drich_n.Geom.source_region.Upper.Z = 5.5
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich_n.GeomInput.concen_region_input.InputType = 'Box'
-drich_n.GeomInput.concen_region_input.GeomName = 'concen_region'
+drich_n.GeomInput.concen_region_input.InputType = "Box"
+drich_n.GeomInput.concen_region_input.GeomName = "concen_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich_n.Geom.concen_region.Lower.X = 60.0
 drich_n.Geom.concen_region.Lower.Y = 80.0
@@ -119,228 +124,264 @@
 drich_n.Geom.concen_region.Upper.Y = 100.0
 drich_n.Geom.concen_region.Upper.Z = 6.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich_n.Geom.Perm.Names = 'background'
+drich_n.Geom.Perm.Names = "background"
 
-drich_n.Geom.background.Perm.Type = 'Constant'
+drich_n.Geom.background.Perm.Type = "Constant"
 drich_n.Geom.background.Perm.Value = 4.0
 
-drich_n.Perm.TensorType = 'TensorByGeom'
+drich_n.Perm.TensorType = "TensorByGeom"
 
-drich_n.Geom.Perm.TensorByGeom.Names = 'background'
+drich_n.Geom.Perm.TensorByGeom.Names = "background"
 
 drich_n.Geom.background.Perm.TensorValX = 1.0
 drich_n.Geom.background.Perm.TensorValY = 1.0
 drich_n.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich_n.SpecificStorage.Type = 'Constant'
-drich_n.SpecificStorage.GeomNames = 'domain'
+drich_n.SpecificStorage.Type = "Constant"
+drich_n.SpecificStorage.GeomNames = "domain"
 drich_n.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich_n.Phase.Names = 'water'
+drich_n.Phase.Names = "water"
 
-drich_n.Phase.water.Density.Type = 'Constant'
+drich_n.Phase.water.Density.Type = "Constant"
 drich_n.Phase.water.Density.Value = 1.0
 
-drich_n.Phase.water.Viscosity.Type = 'Constant'
+drich_n.Phase.water.Viscosity.Type = "Constant"
 drich_n.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich_n.Contaminants.Names = ''
+drich_n.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich_n.Geom.Retardation.GeomNames = ''
+drich_n.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich_n.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich_n.TimingInfo.BaseUnit = 1.0
 drich_n.TimingInfo.StartCount = 0
 drich_n.TimingInfo.StartTime = 0.0
 drich_n.TimingInfo.StopTime = 0.010
 drich_n.TimingInfo.DumpInterval = -1
-drich_n.TimeStep.Type = 'Constant'
+drich_n.TimeStep.Type = "Constant"
 drich_n.TimeStep.Value = 0.001
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich_n.Geom.Porosity.GeomNames = 'background'
+drich_n.Geom.Porosity.GeomNames = "background"
 
-drich_n.Geom.background.Porosity.Type = 'Constant'
+drich_n.Geom.background.Porosity.Type = "Constant"
 drich_n.Geom.background.Porosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich_n.Domain.GeomName = 'domain'
+drich_n.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich_n.Phase.RelPerm.Type = 'VanGenuchten'
-drich_n.Phase.RelPerm.GeomNames = 'domain'
+drich_n.Phase.RelPerm.Type = "VanGenuchten"
+drich_n.Phase.RelPerm.GeomNames = "domain"
 drich_n.Geom.domain.RelPerm.Alpha = 0.005
 drich_n.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich_n.Phase.Saturation.Type = 'VanGenuchten'
-drich_n.Phase.Saturation.GeomNames = 'domain'
+drich_n.Phase.Saturation.Type = "VanGenuchten"
+drich_n.Phase.Saturation.GeomNames = "domain"
 drich_n.Geom.domain.Saturation.Alpha = 0.005
 drich_n.Geom.domain.Saturation.N = 2.0
 drich_n.Geom.domain.Saturation.SRes = 0.2
 drich_n.Geom.domain.Saturation.SSat = 0.99
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich_n.Wells.Names = ''
+drich_n.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich_n.Cycle.Names = 'constant'
-drich_n.Cycle.constant.Names = 'alltime'
+drich_n.Cycle.Names = "constant"
+drich_n.Cycle.constant.Names = "alltime"
 drich_n.Cycle.constant.alltime.Length = 1
 drich_n.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich_n.BCPressure.PatchNames = 'left right front back bottom top'
+drich_n.BCPressure.PatchNames = "left right front back bottom top"
 
-drich_n.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-drich_n.Patch.left.BCPressure.Cycle = 'constant'
-drich_n.Patch.left.BCPressure.RefGeom = 'domain'
-drich_n.Patch.left.BCPressure.RefPatch = 'bottom'
+drich_n.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+drich_n.Patch.left.BCPressure.Cycle = "constant"
+drich_n.Patch.left.BCPressure.RefGeom = "domain"
+drich_n.Patch.left.BCPressure.RefPatch = "bottom"
 drich_n.Patch.left.BCPressure.alltime.Value = 5.0
 
-drich_n.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-drich_n.Patch.right.BCPressure.Cycle = 'constant'
-drich_n.Patch.right.BCPressure.RefGeom = 'domain'
-drich_n.Patch.right.BCPressure.RefPatch = 'bottom'
+drich_n.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+drich_n.Patch.right.BCPressure.Cycle = "constant"
+drich_n.Patch.right.BCPressure.RefGeom = "domain"
+drich_n.Patch.right.BCPressure.RefPatch = "bottom"
 drich_n.Patch.right.BCPressure.alltime.Value = 3.0
 
-drich_n.Patch.front.BCPressure.Type = 'FluxConst'
-drich_n.Patch.front.BCPressure.Cycle = 'constant'
+drich_n.Patch.front.BCPressure.Type = "FluxConst"
+drich_n.Patch.front.BCPressure.Cycle = "constant"
 drich_n.Patch.front.BCPressure.alltime.Value = 0.0
 
-drich_n.Patch.back.BCPressure.Type = 'FluxConst'
-drich_n.Patch.back.BCPressure.Cycle = 'constant'
+drich_n.Patch.back.BCPressure.Type = "FluxConst"
+drich_n.Patch.back.BCPressure.Cycle = "constant"
 drich_n.Patch.back.BCPressure.alltime.Value = 0.0
 
-drich_n.Patch.bottom.BCPressure.Type = 'FluxConst'
-drich_n.Patch.bottom.BCPressure.Cycle = 'constant'
+drich_n.Patch.bottom.BCPressure.Type = "FluxConst"
+drich_n.Patch.bottom.BCPressure.Cycle = "constant"
 drich_n.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-drich_n.Patch.top.BCPressure.Type = 'FluxConst'
-drich_n.Patch.top.BCPressure.Cycle = 'constant'
+drich_n.Patch.top.BCPressure.Type = "FluxConst"
+drich_n.Patch.top.BCPressure.Cycle = "constant"
 drich_n.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich_n.TopoSlopesX.Type = 'Constant'
-drich_n.TopoSlopesX.GeomNames = 'domain'
+drich_n.TopoSlopesX.Type = "Constant"
+drich_n.TopoSlopesX.GeomNames = "domain"
 drich_n.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich_n.TopoSlopesY.Type = 'Constant'
-drich_n.TopoSlopesY.GeomNames = 'domain'
+drich_n.TopoSlopesY.Type = "Constant"
+drich_n.TopoSlopesY.GeomNames = "domain"
 drich_n.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-drich_n.Mannings.Type = 'Constant'
-drich_n.Mannings.GeomNames = 'domain'
-drich_n.Mannings.Geom.domain.Value = 0.
+drich_n.Mannings.Type = "Constant"
+drich_n.Mannings.GeomNames = "domain"
+drich_n.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich_n.ICPressure.Type = 'HydroStaticPatch'
-drich_n.ICPressure.GeomNames = 'domain'
+drich_n.ICPressure.Type = "HydroStaticPatch"
+drich_n.ICPressure.GeomNames = "domain"
 drich_n.Geom.domain.ICPressure.Value = 3.0
-drich_n.Geom.domain.ICPressure.RefGeom = 'domain'
-drich_n.Geom.domain.ICPressure.RefPatch = 'bottom'
+drich_n.Geom.domain.ICPressure.RefGeom = "domain"
+drich_n.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich_n.PhaseSources.water.Type = 'Constant'
-drich_n.PhaseSources.water.GeomNames = 'background'
+drich_n.PhaseSources.water.Type = "Constant"
+drich_n.PhaseSources.water.GeomNames = "background"
 drich_n.PhaseSources.water.Geom.background.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich_n.KnownSolution = 'NoKnownSolution'
+drich_n.KnownSolution = "NoKnownSolution"
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich_n.Solver = 'Richards'
+drich_n.Solver = "Richards"
 drich_n.Solver.MaxIter = 5
 
 drich_n.Solver.Nonlinear.MaxIter = 10
 drich_n.Solver.Nonlinear.ResidualTol = 1e-9
-drich_n.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+drich_n.Solver.Nonlinear.EtaChoice = "EtaConstant"
 drich_n.Solver.Nonlinear.EtaValue = 1e-5
 drich_n.Solver.Nonlinear.UseJacobian = True
 drich_n.Solver.Nonlinear.DerivativeEpsilon = 1e-2
 
 drich_n.Solver.Linear.KrylovDimension = 10
 
-drich_n.Solver.Linear.Preconditioner = 'PFMG'
+drich_n.Solver.Linear.Preconditioner = "PFMG"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/drich_n')
-mkdir(dir_name)
-drich_n.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/default_richards")
+mkdir(new_output_dir_name)
+correct_output_dir_name = get_absolute_path("../correct_output")
+drich_n.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+for i in range(6):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/default_richards_restart.py b/test/python/default_richards_restart.py
new file mode 100644
index 000000000..b5d1b6902
--- /dev/null
+++ b/test/python/default_richards_restart.py
@@ -0,0 +1,434 @@
+# ---------------------------------------------------------
+#  This runs the basic default_richards test case.
+#  This run, as written in this input file, should take
+#  3 nonlinear iterations.
+# ---------------------------------------------------------
+
+import sys, argparse
+from parflow import Run
+from parflow.tools.fs import mkdir, get_absolute_path, rm, cp
+from parflow.tools.compare import pf_test_file, pf_test_file_with_abs
+
+run_name = "default_richards"
+
+default_richards = Run(run_name, __file__)
+
+# ---------------------------------------------------------
+parser = argparse.ArgumentParser()
+parser.add_argument("-p", "--p", default=1)
+parser.add_argument("-q", "--q", default=1)
+parser.add_argument("-r", "--r", default=1)
+args = parser.parse_args()
+
+default_richards.Process.Topology.P = args.p
+default_richards.Process.Topology.Q = args.q
+default_richards.Process.Topology.R = args.r
+
+# ---------------------------------------------------------
+# Computational Grid
+# ---------------------------------------------------------
+
+default_richards.ComputationalGrid.Lower.X = -10.0
+default_richards.ComputationalGrid.Lower.Y = 10.0
+default_richards.ComputationalGrid.Lower.Z = 1.0
+
+default_richards.ComputationalGrid.DX = 8.8888888888888893
+default_richards.ComputationalGrid.DY = 10.666666666666666
+default_richards.ComputationalGrid.DZ = 1.0
+
+default_richards.ComputationalGrid.NX = 18
+default_richards.ComputationalGrid.NY = 15
+default_richards.ComputationalGrid.NZ = 8
+
+# ---------------------------------------------------------
+# The Names of the GeomInputs
+# ---------------------------------------------------------
+
+default_richards.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
+
+# ---------------------------------------------------------
+# Domain Geometry Input
+# ---------------------------------------------------------
+
+default_richards.GeomInput.domain_input.InputType = "Box"
+default_richards.GeomInput.domain_input.GeomName = "domain"
+
+# ---------------------------------------------------------
+# Domain Geometry
+# ---------------------------------------------------------
+
+default_richards.Geom.domain.Lower.X = -10.0
+default_richards.Geom.domain.Lower.Y = 10.0
+default_richards.Geom.domain.Lower.Z = 1.0
+
+default_richards.Geom.domain.Upper.X = 150.0
+default_richards.Geom.domain.Upper.Y = 170.0
+default_richards.Geom.domain.Upper.Z = 9.0
+
+default_richards.Geom.domain.Patches = "left right front back bottom top"
+
+# ---------------------------------------------------------
+# Background Geometry Input
+# ---------------------------------------------------------
+
+default_richards.GeomInput.background_input.InputType = "Box"
+default_richards.GeomInput.background_input.GeomName = "background"
+
+# ---------------------------------------------------------
+# Background Geometry
+# ---------------------------------------------------------
+
+default_richards.Geom.background.Lower.X = -99999999.0
+default_richards.Geom.background.Lower.Y = -99999999.0
+default_richards.Geom.background.Lower.Z = -99999999.0
+
+default_richards.Geom.background.Upper.X = 99999999.0
+default_richards.Geom.background.Upper.Y = 99999999.0
+default_richards.Geom.background.Upper.Z = 99999999.0
+
+# ---------------------------------------------------------
+# Source_Region Geometry Input
+# ---------------------------------------------------------
+
+default_richards.GeomInput.source_region_input.InputType = "Box"
+default_richards.GeomInput.source_region_input.GeomName = "source_region"
+
+# ---------------------------------------------------------
+# Source_Region Geometry
+# ---------------------------------------------------------
+
+default_richards.Geom.source_region.Lower.X = 65.56
+default_richards.Geom.source_region.Lower.Y = 79.34
+default_richards.Geom.source_region.Lower.Z = 4.5
+
+default_richards.Geom.source_region.Upper.X = 74.44
+default_richards.Geom.source_region.Upper.Y = 89.99
+default_richards.Geom.source_region.Upper.Z = 5.5
+
+# ---------------------------------------------------------
+# Concen_Region Geometry Input
+# ---------------------------------------------------------
+
+default_richards.GeomInput.concen_region_input.InputType = "Box"
+default_richards.GeomInput.concen_region_input.GeomName = "concen_region"
+
+# ---------------------------------------------------------
+# Concen_Region Geometry
+# ---------------------------------------------------------
+
+default_richards.Geom.concen_region.Lower.X = 60.0
+default_richards.Geom.concen_region.Lower.Y = 80.0
+default_richards.Geom.concen_region.Lower.Z = 4.0
+
+default_richards.Geom.concen_region.Upper.X = 80.0
+default_richards.Geom.concen_region.Upper.Y = 100.0
+default_richards.Geom.concen_region.Upper.Z = 6.0
+
+# -----------------------------------------------------------------------------
+# Perm
+# -----------------------------------------------------------------------------
+
+default_richards.Geom.Perm.Names = "background"
+
+default_richards.Geom.background.Perm.Type = "Constant"
+default_richards.Geom.background.Perm.Value = 4.0
+
+default_richards.Perm.TensorType = "TensorByGeom"
+
+default_richards.Geom.Perm.TensorByGeom.Names = "background"
+
+default_richards.Geom.background.Perm.TensorValX = 1.0
+default_richards.Geom.background.Perm.TensorValY = 1.0
+default_richards.Geom.background.Perm.TensorValZ = 1.0
+
+# -----------------------------------------------------------------------------
+# Specific Storage
+# -----------------------------------------------------------------------------
+
+default_richards.SpecificStorage.Type = "Constant"
+default_richards.SpecificStorage.GeomNames = "domain"
+default_richards.Geom.domain.SpecificStorage.Value = 1.0e-4
+
+# -----------------------------------------------------------------------------
+# Phases
+# -----------------------------------------------------------------------------
+
+default_richards.Phase.Names = "water"
+
+default_richards.Phase.water.Density.Type = "Constant"
+default_richards.Phase.water.Density.Value = 1.0
+
+default_richards.Phase.water.Viscosity.Type = "Constant"
+default_richards.Phase.water.Viscosity.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Contaminants
+# -----------------------------------------------------------------------------
+
+default_richards.Contaminants.Names = ""
+
+# -----------------------------------------------------------------------------
+# Retardation
+# -----------------------------------------------------------------------------
+
+default_richards.Geom.Retardation.GeomNames = ""
+
+# -----------------------------------------------------------------------------
+# Gravity
+# -----------------------------------------------------------------------------
+
+default_richards.Gravity = 1.0
+
+# -----------------------------------------------------------------------------
+# Setup timing info
+# -----------------------------------------------------------------------------
+
+default_richards.TimingInfo.BaseUnit = 1.0
+default_richards.TimingInfo.StartCount = 25
+default_richards.TimingInfo.StartTime = 0.0
+default_richards.TimingInfo.StopTime = 1.0
+default_richards.TimingInfo.DumpInterval = -1
+default_richards.TimeStep.Type = "Constant"
+default_richards.TimeStep.Value = 0.001
+
+# -----------------------------------------------------------------------------
+# Porosity
+# -----------------------------------------------------------------------------
+
+default_richards.Geom.Porosity.GeomNames = "background"
+default_richards.Geom.background.Porosity.Type = "Constant"
+default_richards.Geom.background.Porosity.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Domain
+# -----------------------------------------------------------------------------
+
+default_richards.Domain.GeomName = "domain"
+
+# -----------------------------------------------------------------------------
+# Relative Permeability
+# -----------------------------------------------------------------------------
+
+default_richards.Phase.RelPerm.Type = "VanGenuchten"
+default_richards.Phase.RelPerm.GeomNames = "domain"
+default_richards.Geom.domain.RelPerm.Alpha = 0.005
+default_richards.Geom.domain.RelPerm.N = 2.0
+
+# ---------------------------------------------------------
+# Saturation
+# ---------------------------------------------------------
+
+default_richards.Phase.Saturation.Type = "VanGenuchten"
+default_richards.Phase.Saturation.GeomNames = "domain"
+default_richards.Geom.domain.Saturation.Alpha = 0.005
+default_richards.Geom.domain.Saturation.N = 2.0
+default_richards.Geom.domain.Saturation.SRes = 0.2
+default_richards.Geom.domain.Saturation.SSat = 0.99
+
+# -----------------------------------------------------------------------------
+# Wells
+# -----------------------------------------------------------------------------
+
+default_richards.Wells.Names = ""
+
+# -----------------------------------------------------------------------------
+# Time Cycles
+# -----------------------------------------------------------------------------
+
+default_richards.Cycle.Names = "constant"
+default_richards.Cycle.constant.Names = "alltime"
+default_richards.Cycle.constant.alltime.Length = 1
+default_richards.Cycle.constant.Repeat = -1
+
+# -----------------------------------------------------------------------------
+# Boundary Conditions: Pressure
+# -----------------------------------------------------------------------------
+
+default_richards.BCPressure.PatchNames = "left right front back bottom top"
+
+default_richards.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+default_richards.Patch.left.BCPressure.Cycle = "constant"
+default_richards.Patch.left.BCPressure.RefGeom = "domain"
+default_richards.Patch.left.BCPressure.RefPatch = "bottom"
+default_richards.Patch.left.BCPressure.alltime.Value = 5.0
+
+default_richards.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+default_richards.Patch.right.BCPressure.Cycle = "constant"
+default_richards.Patch.right.BCPressure.RefGeom = "domain"
+default_richards.Patch.right.BCPressure.RefPatch = "bottom"
+default_richards.Patch.right.BCPressure.alltime.Value = 3.0
+
+default_richards.Patch.front.BCPressure.Type = "FluxConst"
+default_richards.Patch.front.BCPressure.Cycle = "constant"
+default_richards.Patch.front.BCPressure.alltime.Value = 0.0
+
+default_richards.Patch.back.BCPressure.Type = "FluxConst"
+default_richards.Patch.back.BCPressure.Cycle = "constant"
+default_richards.Patch.back.BCPressure.alltime.Value = 0.0
+
+default_richards.Patch.bottom.BCPressure.Type = "FluxConst"
+default_richards.Patch.bottom.BCPressure.Cycle = "constant"
+default_richards.Patch.bottom.BCPressure.alltime.Value = 0.0
+
+default_richards.Patch.top.BCPressure.Type = "FluxConst"
+default_richards.Patch.top.BCPressure.Cycle = "constant"
+default_richards.Patch.top.BCPressure.alltime.Value = 0.0
+
+# ---------------------------------------------------------
+# Topo slopes in x-direction
+# ---------------------------------------------------------
+
+default_richards.TopoSlopesX.Type = "Constant"
+default_richards.TopoSlopesX.GeomNames = "domain"
+default_richards.TopoSlopesX.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Topo slopes in y-direction
+# ---------------------------------------------------------
+
+default_richards.TopoSlopesY.Type = "Constant"
+default_richards.TopoSlopesY.GeomNames = "domain"
+default_richards.TopoSlopesY.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
+
+default_richards.Mannings.Type = "Constant"
+default_richards.Mannings.GeomNames = "domain"
+default_richards.Mannings.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Initial conditions: water pressure
+# ---------------------------------------------------------
+
+# default_richards.ICPressure.Type = 'HydroStaticPatch'
+# default_richards.ICPressure.GeomNames = 'domain'
+# default_richards.Geom.domain.ICPressure.Value = 3.0
+# default_richards.Geom.domain.ICPressure.RefGeom = 'domain'
+# default_richards.Geom.domain.ICPressure.RefPatch = 'bottom'
+
+default_richards.ICPressure.Type = "PFBFile"
+default_richards.ICPressure.GeomNames = "domain"
+default_richards.Geom.domain.ICPressure.RefPatch = "top"
+default_richards.Geom.domain.ICPressure.FileName = (
+    "default_richards.out.press.00025.pfb"
+)
+
+# -----------------------------------------------------------------------------
+# Phase sources:
+# -----------------------------------------------------------------------------
+
+default_richards.PhaseSources.water.Type = "Constant"
+default_richards.PhaseSources.water.GeomNames = "background"
+default_richards.PhaseSources.water.Geom.background.Value = 0.0
+
+# -----------------------------------------------------------------------------
+# Exact solution specification for error calculations
+# -----------------------------------------------------------------------------
+
+default_richards.KnownSolution = "NoKnownSolution"
+
+# -----------------------------------------------------------------------------
+# Set solver parameters
+# -----------------------------------------------------------------------------
+
+default_richards.Solver = "Richards"
+default_richards.Solver.MaxIter = 50
+
+default_richards.Solver.Nonlinear.MaxIter = 10
+default_richards.Solver.Nonlinear.ResidualTol = 1e-9
+default_richards.Solver.Nonlinear.EtaChoice = "EtaConstant"
+default_richards.Solver.Nonlinear.EtaValue = 1e-5
+default_richards.Solver.Nonlinear.UseJacobian = True
+default_richards.Solver.Nonlinear.DerivativeEpsilon = 1e-2
+
+default_richards.Solver.Linear.KrylovDimension = 10
+
+default_richards.Solver.Linear.Preconditioner = "PFMG"
+
+default_richards.Solver.PrintVelocities = True
+
+# ---------------------------------------------------------
+# Copy restart pressure
+# ---------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/default_richards_restart")
+mkdir(new_output_dir_name)
+cp("../correct_output/default_richards.out.press.00025.pfb", new_output_dir_name)
+
+default_richards.dist(new_output_dir_name + "/default_richards.out.press.00025.pfb")
+
+# -----------------------------------------------------------------------------
+# Run ParFlow
+# -----------------------------------------------------------------------------
+
+correct_output_dir_name = get_absolute_path("../correct_output")
+default_richards.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+
+sig_digits = 6
+abs_value = 1e-12
+for i in range(26, 51):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.velx.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in x-velocity for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+    filename = f"/{run_name}.out.vely.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in y-velocity for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+    filename = f"/{run_name}.out.vely.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in z-velocity for timestep {timestep}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/default_richards_restart_netcdf.py b/test/python/default_richards_restart_netcdf.py
new file mode 100644
index 000000000..3a9197c73
--- /dev/null
+++ b/test/python/default_richards_restart_netcdf.py
@@ -0,0 +1,583 @@
+#  Testing restart using PFB and NetCDF files
+
+import sys, argparse, shutil, os, glob
+from parflow import Run
+from parflow.tools.fs import mkdir, get_absolute_path, rm
+from parflow.tools.compare import pf_test_file, pf_test_file_with_abs
+
+run_name = "default_richards"
+
+default_richards = Run(run_name, __file__)
+
+
+def remove_restart_files():
+    """
+    Remove restart check files between the restart tests.
+    """
+    for path in glob.glob("default_richards_restart.out.press.*") + glob.glob(
+        "default_richards_restart.out.satur.*"
+    ):
+        os.remove(path)
+
+
+default_richards.FileVersion = 4
+
+# ---------------------------------------------------------
+parser = argparse.ArgumentParser()
+parser.add_argument("-p", "--p", default=1)
+parser.add_argument("-q", "--q", default=1)
+parser.add_argument("-r", "--r", default=1)
+args = parser.parse_args()
+
+default_richards.Process.Topology.P = args.p
+default_richards.Process.Topology.Q = args.q
+default_richards.Process.Topology.R = args.r
+
+# ---------------------------------------------------------
+# Computational Grid
+# ---------------------------------------------------------
+default_richards.ComputationalGrid.Lower.X = -10.0
+default_richards.ComputationalGrid.Lower.Y = 10.0
+default_richards.ComputationalGrid.Lower.Z = 1.0
+
+default_richards.ComputationalGrid.DX = 8.8888888888888893
+default_richards.ComputationalGrid.DY = 10.666666666666666
+default_richards.ComputationalGrid.DZ = 1.0
+
+default_richards.ComputationalGrid.NX = 18
+default_richards.ComputationalGrid.NY = 15
+default_richards.ComputationalGrid.NZ = 8
+
+# ---------------------------------------------------------
+# The Names of the GeomInputs
+# ---------------------------------------------------------
+default_richards.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
+
+# ---------------------------------------------------------
+# Domain Geometry Input
+# ---------------------------------------------------------
+default_richards.GeomInput.domain_input.InputType = "Box"
+default_richards.GeomInput.domain_input.GeomName = "domain"
+
+# ---------------------------------------------------------
+# Domain Geometry
+# ---------------------------------------------------------
+default_richards.Geom.domain.Lower.X = -10.0
+default_richards.Geom.domain.Lower.Y = 10.0
+default_richards.Geom.domain.Lower.Z = 1.0
+
+default_richards.Geom.domain.Upper.X = 150.0
+default_richards.Geom.domain.Upper.Y = 170.0
+default_richards.Geom.domain.Upper.Z = 9.0
+
+default_richards.Geom.domain.Patches = "left right front back bottom top"
+
+# ---------------------------------------------------------
+# Background Geometry Input
+# ---------------------------------------------------------
+default_richards.GeomInput.background_input.InputType = "Box"
+default_richards.GeomInput.background_input.GeomName = "background"
+
+# ---------------------------------------------------------
+# Background Geometry
+# ---------------------------------------------------------
+default_richards.Geom.background.Lower.X = -99999999.0
+default_richards.Geom.background.Lower.Y = -99999999.0
+default_richards.Geom.background.Lower.Z = -99999999.0
+
+default_richards.Geom.background.Upper.X = 99999999.0
+default_richards.Geom.background.Upper.Y = 99999999.0
+default_richards.Geom.background.Upper.Z = 99999999.0
+
+
+# ---------------------------------------------------------
+# Source_Region Geometry Input
+# ---------------------------------------------------------
+default_richards.GeomInput.source_region_input.InputType = "Box"
+default_richards.GeomInput.source_region_input.GeomName = "source_region"
+
+# ---------------------------------------------------------
+# Source_Region Geometry
+# ---------------------------------------------------------
+default_richards.Geom.source_region.Lower.X = 65.56
+default_richards.Geom.source_region.Lower.Y = 79.34
+default_richards.Geom.source_region.Lower.Z = 4.5
+
+default_richards.Geom.source_region.Upper.X = 74.44
+default_richards.Geom.source_region.Upper.Y = 89.99
+default_richards.Geom.source_region.Upper.Z = 5.5
+
+
+# ---------------------------------------------------------
+# Concen_Region Geometry Input
+# ---------------------------------------------------------
+default_richards.GeomInput.concen_region_input.InputType = "Box"
+default_richards.GeomInput.concen_region_input.GeomName = "concen_region"
+
+# ---------------------------------------------------------
+# Concen_Region Geometry
+# ---------------------------------------------------------
+default_richards.Geom.concen_region.Lower.X = 60.0
+default_richards.Geom.concen_region.Lower.Y = 80.0
+default_richards.Geom.concen_region.Lower.Z = 4.0
+
+default_richards.Geom.concen_region.Upper.X = 80.0
+default_richards.Geom.concen_region.Upper.Y = 100.0
+default_richards.Geom.concen_region.Upper.Z = 6.0
+
+# -----------------------------------------------------------------------------
+# Perm
+# -----------------------------------------------------------------------------
+default_richards.Geom.Perm.Names = "background"
+
+default_richards.Geom.background.Perm.Type = "Constant"
+default_richards.Geom.background.Perm.Value = 4.0
+
+default_richards.Perm.TensorType = "TensorByGeom"
+
+default_richards.Geom.Perm.TensorByGeom.Names = "background"
+
+default_richards.Geom.background.Perm.TensorValX = 1.0
+default_richards.Geom.background.Perm.TensorValY = 1.0
+default_richards.Geom.background.Perm.TensorValZ = 1.0
+
+# -----------------------------------------------------------------------------
+# Specific Storage
+# -----------------------------------------------------------------------------
+
+default_richards.SpecificStorage.Type = "Constant"
+default_richards.SpecificStorage.GeomNames = "domain"
+default_richards.Geom.domain.SpecificStorage.Value = 1.0e-4
+
+# -----------------------------------------------------------------------------
+# Phases
+# -----------------------------------------------------------------------------
+
+default_richards.Phase.Names = "water"
+
+default_richards.Phase.water.Density.Type = "Constant"
+default_richards.Phase.water.Density.Value = 1.0
+
+default_richards.Phase.water.Viscosity.Type = "Constant"
+default_richards.Phase.water.Viscosity.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Contaminants
+# -----------------------------------------------------------------------------
+default_richards.Contaminants.Names = ""
+
+# -----------------------------------------------------------------------------
+# Retardation
+# -----------------------------------------------------------------------------
+default_richards.Geom.Retardation.GeomNames = ""
+
+# -----------------------------------------------------------------------------
+# Gravity
+# -----------------------------------------------------------------------------
+
+default_richards.Gravity = 1.0
+
+# -----------------------------------------------------------------------------
+# Setup timing info
+# -----------------------------------------------------------------------------
+
+timestep = 0.001
+
+default_richards.TimingInfo.BaseUnit = 1.0
+default_richards.TimingInfo.StartCount = 0
+default_richards.TimingInfo.StartTime = 0.0
+default_richards.TimingInfo.StopTime = 0.05
+default_richards.TimingInfo.DumpInterval = -1
+default_richards.TimeStep.Type = "Constant"
+default_richards.TimeStep.Value = timestep
+
+# -----------------------------------------------------------------------------
+# Porosity
+# -----------------------------------------------------------------------------
+
+default_richards.Geom.Porosity.GeomNames = "background"
+
+default_richards.Geom.background.Porosity.Type = "Constant"
+default_richards.Geom.background.Porosity.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Domain
+# -----------------------------------------------------------------------------
+default_richards.Domain.GeomName = "domain"
+
+# -----------------------------------------------------------------------------
+# Relative Permeability
+# -----------------------------------------------------------------------------
+
+default_richards.Phase.RelPerm.Type = "VanGenuchten"
+default_richards.Phase.RelPerm.GeomNames = "domain"
+default_richards.Geom.domain.RelPerm.Alpha = 0.005
+default_richards.Geom.domain.RelPerm.N = 2.0
+
+# ---------------------------------------------------------
+# Saturation
+# ---------------------------------------------------------
+
+default_richards.Phase.Saturation.Type = "VanGenuchten"
+default_richards.Phase.Saturation.GeomNames = "domain"
+default_richards.Geom.domain.Saturation.Alpha = 0.005
+default_richards.Geom.domain.Saturation.N = 2.0
+default_richards.Geom.domain.Saturation.SRes = 0.2
+default_richards.Geom.domain.Saturation.SSat = 0.99
+
+# -----------------------------------------------------------------------------
+# Wells
+# -----------------------------------------------------------------------------
+default_richards.Wells.Names = ""
+
+# -----------------------------------------------------------------------------
+# Time Cycles
+# -----------------------------------------------------------------------------
+default_richards.Cycle.Names = "constant"
+default_richards.Cycle.constant.Names = "alltime"
+default_richards.Cycle.constant.alltime.Length = 1
+default_richards.Cycle.constant.Repeat = -1
+
+# -----------------------------------------------------------------------------
+# Boundary Conditions: Pressure
+# -----------------------------------------------------------------------------
+default_richards.BCPressure.PatchNames = "left right front back bottom top"
+
+default_richards.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+default_richards.Patch.left.BCPressure.Cycle = "constant"
+default_richards.Patch.left.BCPressure.RefGeom = "domain"
+default_richards.Patch.left.BCPressure.RefPatch = "bottom"
+default_richards.Patch.left.BCPressure.alltime.Value = 5.0
+
+default_richards.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+default_richards.Patch.right.BCPressure.Cycle = "constant"
+default_richards.Patch.right.BCPressure.RefGeom = "domain"
+default_richards.Patch.right.BCPressure.RefPatch = "bottom"
+default_richards.Patch.right.BCPressure.alltime.Value = 3.0
+
+default_richards.Patch.front.BCPressure.Type = "FluxConst"
+default_richards.Patch.front.BCPressure.Cycle = "constant"
+default_richards.Patch.front.BCPressure.alltime.Value = 0.0
+
+default_richards.Patch.back.BCPressure.Type = "FluxConst"
+default_richards.Patch.back.BCPressure.Cycle = "constant"
+default_richards.Patch.back.BCPressure.alltime.Value = 0.0
+
+default_richards.Patch.bottom.BCPressure.Type = "FluxConst"
+default_richards.Patch.bottom.BCPressure.Cycle = "constant"
+default_richards.Patch.bottom.BCPressure.alltime.Value = 0.0
+
+default_richards.Patch.top.BCPressure.Type = "FluxConst"
+default_richards.Patch.top.BCPressure.Cycle = "constant"
+default_richards.Patch.top.BCPressure.alltime.Value = 0.0
+
+# ---------------------------------------------------------
+# Topo slopes in x-direction
+# ---------------------------------------------------------
+
+default_richards.TopoSlopesX.Type = "Constant"
+default_richards.TopoSlopesX.GeomNames = ""
+
+default_richards.TopoSlopesX.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Topo slopes in y-direction
+# ---------------------------------------------------------
+
+default_richards.TopoSlopesY.Type = "Constant"
+default_richards.TopoSlopesY.GeomNames = ""
+
+default_richards.TopoSlopesY.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
+
+default_richards.Mannings.Type = "Constant"
+default_richards.Mannings.GeomNames = ""
+default_richards.Mannings.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Initial conditions: water pressure
+# ---------------------------------------------------------
+
+default_richards.ICPressure.Type = "HydroStaticPatch"
+default_richards.ICPressure.GeomNames = "domain"
+default_richards.Geom.domain.ICPressure.Value = 3.0
+default_richards.Geom.domain.ICPressure.RefGeom = "domain"
+default_richards.Geom.domain.ICPressure.RefPatch = "bottom"
+
+# -----------------------------------------------------------------------------
+# Phase sources:
+# -----------------------------------------------------------------------------
+
+default_richards.PhaseSources.water.Type = "Constant"
+default_richards.PhaseSources.water.GeomNames = "background"
+default_richards.PhaseSources.water.Geom.background.Value = 0.0
+
+
+# -----------------------------------------------------------------------------
+# Exact solution specification for error calculations
+# -----------------------------------------------------------------------------
+
+default_richards.KnownSolution = "NoKnownSolution"
+
+
+# -----------------------------------------------------------------------------
+# Set solver parameters
+# -----------------------------------------------------------------------------
+default_richards.Solver = "Richards"
+default_richards.Solver.MaxIter = 50
+
+default_richards.Solver.Nonlinear.MaxIter = 10
+default_richards.Solver.Nonlinear.ResidualTol = 1e-9
+default_richards.Solver.Nonlinear.EtaChoice = "EtaConstant"
+default_richards.Solver.Nonlinear.EtaValue = 1e-5
+default_richards.Solver.Nonlinear.UseJacobian = True
+default_richards.Solver.Nonlinear.DerivativeEpsilon = 1e-2
+
+default_richards.Solver.Linear.KrylovDimension = 10
+
+default_richards.Solver.Linear.Preconditioner = "PFMG"
+
+default_richards.Solver.PrintVelocities = True
+
+# -----------------------------------------------------------------------------
+# Write pressure as NetCDF for testing restart
+# -----------------------------------------------------------------------------
+default_richards.NetCDF.WritePressure = True
+default_richards.NetCDF.NumStepsPerFile = 25
+
+# -----------------------------------------------------------------------------
+# Run and Unload the ParFlow output files
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/default_richards")
+mkdir(new_output_dir_name)
+correct_output_dir_name = get_absolute_path("../correct_output")
+default_richards.run(working_directory=new_output_dir_name)
+
+#
+# Tests
+#
+# source pftest.tcl
+passed = True
+
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+sig_digits = 6
+abs_value = 1e-12
+for i in range(51):
+    timeindex = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timeindex}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timeindex {timeindex}",
+    ):
+        passed = False
+
+    filename = f"/{run_name}.out.satur.{timeindex}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timeindex {timeindex}",
+    ):
+        passed = False
+
+    filename = f"/{run_name}.out.vely.{timeindex}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in y-velocity for timeindex {timeindex}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+    filename = f"/{run_name}.out.vely.{timeindex}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in z-velocity for timeindex {timeindex}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
+
+
+passed = 1
+remove_restart_files()
+
+# Restart run from step 10
+istep = 10
+initial_pressure = "initial_pressure.pfb"
+shutil.copy(
+    os.path.join(new_output_dir_name, "default_richards.out.press.00010.pfb"),
+    os.path.join(new_output_dir_name, initial_pressure),
+)
+
+default_richards.dist(os.path.join(new_output_dir_name, initial_pressure))
+
+default_richards.Solver.PrintInitialConditions = False
+
+default_richards.ICPressure.Type = "PFBFile"
+default_richards.ICPressure.GeomNames = "domain"
+default_richards.Geom.domain.ICPressure.FileName = initial_pressure
+default_richards.Geom.domain.ICPressure.RefGeom = "domain"
+default_richards.Geom.domain.ICPressure.RefPatch = "bottom"
+
+default_richards.TimingInfo.StartCount = istep
+default_richards.TimingInfo.StartTime = istep * timestep
+
+run_name = "default_richards_restart"
+
+default_richards.set_name(run_name)
+
+default_richards.run(working_directory=new_output_dir_name)
+
+for i in range(11, 51):
+    timeindex = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timeindex}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timeindex {timeindex}",
+    ):
+        passed = False
+
+    filename = f"/{run_name}.out.satur.{timeindex}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timeindex {timeindex}",
+    ):
+        passed = False
+
+# Remove the temporary initial pressure for the restart
+# file delete -force $initial_pressure
+if passed:
+    print("default_richards_restart PFB : PASSED")
+else:
+    print("default_richards_restart PFB : FAILED")
+    sys.exit(1)
+
+# Restart run from step 10
+
+remove_restart_files()
+
+passed = True
+
+istep = 10
+initial_pressure = "default_richards.out.00001.nc"
+
+default_richards.Solver.PrintInitialConditions = False
+
+default_richards.ICPressure.Type = "NCFile"
+default_richards.ICPressure.GeomNames = "domain"
+default_richards.Geom.domain.ICPressure.FileName = initial_pressure
+default_richards.Geom.domain.ICPressure.TimeStep = 9
+default_richards.Geom.domain.ICPressure.RefGeom = "domain"
+default_richards.Geom.domain.ICPressure.RefPatch = "bottom"
+
+
+default_richards.TimingInfo.StartCount = istep
+default_richards.TimingInfo.StartTime = istep * timestep
+
+default_richards.run(working_directory=new_output_dir_name)
+
+for i in range(11, 51):
+    timeindex = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timeindex}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timeindex {timeindex}",
+    ):
+        passed = False
+
+    filename = f"/{run_name}.out.satur.{timeindex}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timeindex {timeindex}",
+    ):
+        passed = False
+
+# Remove the temporary initial pressure for the restart
+# file delete -force $initial_pressure
+if passed:
+    print("default_richards_restart NetCDF : PASSED")
+else:
+    print("default_richards_restart NetCDF : FAILED")
+    sys.exit(1)
+
+# # Restart run from step last timestep in a file.  Indices in the
+# # NetCDF files may be negative values to index from last
+# # timestep.  This is similar to Python vector indexing.
+# # Note we still have to know the initial timestep (istep here)
+# # index but the negative index feature makes restartng
+# # from NetCDF files a bit easier.
+
+remove_restart_files()
+
+passed = True
+
+istep = 25
+initial_pressure = "default_richards.out.00001.nc"
+
+default_richards.Solver.PrintInitialConditions = False
+
+default_richards.ICPressure.Type = "NCFile"
+default_richards.ICPressure.GeomNames = "domain"
+default_richards.Geom.domain.ICPressure.FileName = initial_pressure
+default_richards.Geom.domain.ICPressure.TimeStep = -1
+default_richards.Geom.domain.ICPressure.RefGeom = "domain"
+default_richards.Geom.domain.ICPressure.RefPatch = "bottom"
+
+
+default_richards.TimingInfo.StartCount = istep
+default_richards.TimingInfo.StartTime = istep * timestep
+
+default_richards.run(working_directory=new_output_dir_name)
+
+for i in range(25, 51):
+    timeindex = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timeindex}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timeindex {timeindex}",
+    ):
+        passed = False
+
+    filename = f"/{run_name}.out.satur.{timeindex}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timeindex {timeindex}",
+    ):
+        passed = False
+
+if passed:
+    print("default_richards_restart NetCDF negative index: PASSED")
+else:
+    print("default_richards_restart NetCDF negative index: FAILED")
+    sys.exit(1)
diff --git a/test/python/default_richards_wells.py b/test/python/default_richards_wells.py
index dfe9ba68a..4ad6e9550 100644
--- a/test/python/default_richards_wells.py
+++ b/test/python/default_richards_wells.py
@@ -1,25 +1,26 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 #  This runs the basic default_richards test case.
 #  This run, as written in this input file, should take
 #  3 nonlinear iterations.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
-drich = Run("default_richards_wells", __file__)
-
-#---------------------------------------------------------
-
+run_name = "default_richards_wells"
+drich = Run(run_name, __file__)
+# ---------------------------------------------------------
 drich.FileVersion = 4
 
 drich.Process.Topology.P = 1
 drich.Process.Topology.Q = 1
 drich.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.ComputationalGrid.Lower.X = -10.0
 drich.ComputationalGrid.Lower.Y = 10.0
@@ -33,22 +34,24 @@
 drich.ComputationalGrid.NY = 10
 drich.ComputationalGrid.NZ = 8
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.Names = 'domain_input background_input source_region_input concen_region_input'
+drich.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.domain_input.InputType = 'Box'
-drich.GeomInput.domain_input.GeomName = 'domain'
+drich.GeomInput.domain_input.InputType = "Box"
+drich.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Geom.domain.Lower.X = -10.0
 drich.Geom.domain.Lower.Y = 10.0
@@ -58,18 +61,18 @@
 drich.Geom.domain.Upper.Y = 170.0
 drich.Geom.domain.Upper.Z = 9.0
 
-drich.Geom.domain.Patches = 'left right front back bottom top'
+drich.Geom.domain.Patches = "left right front back bottom top"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.background_input.InputType = 'Box'
-drich.GeomInput.background_input.GeomName = 'background'
+drich.GeomInput.background_input.InputType = "Box"
+drich.GeomInput.background_input.GeomName = "background"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Geom.background.Lower.X = -99999999.0
 drich.Geom.background.Lower.Y = -99999999.0
@@ -79,16 +82,16 @@
 drich.Geom.background.Upper.Y = 99999999.0
 drich.Geom.background.Upper.Z = 99999999.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.source_region_input.InputType = 'Box'
-drich.GeomInput.source_region_input.GeomName = 'source_region'
+drich.GeomInput.source_region_input.InputType = "Box"
+drich.GeomInput.source_region_input.GeomName = "source_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Geom.source_region.Lower.X = 65.56
 drich.Geom.source_region.Lower.Y = 79.34
@@ -98,16 +101,16 @@
 drich.Geom.source_region.Upper.Y = 89.99
 drich.Geom.source_region.Upper.Z = 5.5
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.concen_region_input.InputType = 'Box'
-drich.GeomInput.concen_region_input.GeomName = 'concen_region'
+drich.GeomInput.concen_region_input.InputType = "Box"
+drich.GeomInput.concen_region_input.GeomName = "concen_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Geom.concen_region.Lower.X = 60.0
 drich.Geom.concen_region.Lower.Y = 80.0
@@ -117,238 +120,285 @@
 drich.Geom.concen_region.Upper.Y = 100.0
 drich.Geom.concen_region.Upper.Z = 6.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Geom.Perm.Names = 'background'
+drich.Geom.Perm.Names = "background"
 
-drich.Geom.background.Perm.Type = 'Constant'
+drich.Geom.background.Perm.Type = "Constant"
 drich.Geom.background.Perm.Value = 4.0
 
-drich.Perm.TensorType = 'TensorByGeom'
+drich.Perm.TensorType = "TensorByGeom"
 
-drich.Geom.Perm.TensorByGeom.Names = 'background'
+drich.Geom.Perm.TensorByGeom.Names = "background"
 
 drich.Geom.background.Perm.TensorValX = 1.0
 drich.Geom.background.Perm.TensorValY = 1.0
 drich.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.SpecificStorage.Type = 'Constant'
-drich.SpecificStorage.GeomNames = 'domain'
+drich.SpecificStorage.Type = "Constant"
+drich.SpecificStorage.GeomNames = "domain"
 drich.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Phase.Names = 'water'
+drich.Phase.Names = "water"
 
-drich.Phase.water.Density.Type = 'Constant'
+drich.Phase.water.Density.Type = "Constant"
 drich.Phase.water.Density.Value = 1.0
 
-drich.Phase.water.Viscosity.Type = 'Constant'
+drich.Phase.water.Viscosity.Type = "Constant"
 drich.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Contaminants.Names = ''
+drich.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Geom.Retardation.GeomNames = ''
+drich.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.TimingInfo.BaseUnit = 1.0
 drich.TimingInfo.StartCount = 0
 drich.TimingInfo.StartTime = 0.0
 drich.TimingInfo.StopTime = 0.010
 drich.TimingInfo.DumpInterval = -1
-drich.TimeStep.Type = 'Constant'
+drich.TimeStep.Type = "Constant"
 drich.TimeStep.Value = 0.001
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Geom.Porosity.GeomNames = 'background'
+drich.Geom.Porosity.GeomNames = "background"
 
-drich.Geom.background.Porosity.Type = 'Constant'
+drich.Geom.background.Porosity.Type = "Constant"
 drich.Geom.background.Porosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
-drich.Domain.GeomName = 'domain'
+# -----------------------------------------------------------------------------
+drich.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Phase.RelPerm.Type = 'VanGenuchten'
-drich.Phase.RelPerm.GeomNames = 'domain'
+drich.Phase.RelPerm.Type = "VanGenuchten"
+drich.Phase.RelPerm.GeomNames = "domain"
 drich.Geom.domain.RelPerm.Alpha = 0.005
 drich.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.Phase.Saturation.Type = 'VanGenuchten'
-drich.Phase.Saturation.GeomNames = 'domain'
+drich.Phase.Saturation.Type = "VanGenuchten"
+drich.Phase.Saturation.GeomNames = "domain"
 drich.Geom.domain.Saturation.Alpha = 0.005
 drich.Geom.domain.Saturation.N = 2.0
 drich.Geom.domain.Saturation.SRes = 0.2
 drich.Geom.domain.Saturation.SSat = 0.99
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Cycle.Names = 'constant'
-drich.Cycle.constant.Names = 'alltime'
+drich.Cycle.Names = "constant"
+drich.Cycle.constant.Names = "alltime"
 drich.Cycle.constant.alltime.Length = 1
 drich.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Wells.Names = 'pumping_well'
-drich.Wells.pumping_well.InputType = 'Vertical'
-drich.Wells.pumping_well.Action = 'Extraction'
-drich.Wells.pumping_well.Type = 'Pressure'
+drich.Wells.Names = "pumping_well"
+drich.Wells.pumping_well.InputType = "Vertical"
+drich.Wells.pumping_well.Action = "Extraction"
+drich.Wells.pumping_well.Type = "Pressure"
 drich.Wells.pumping_well.X = 0
 drich.Wells.pumping_well.Y = 80
 drich.Wells.pumping_well.ZUpper = 3.0
 drich.Wells.pumping_well.ZLower = 2.00
-drich.Wells.pumping_well.Method = 'Standard'
-drich.Wells.pumping_well.Cycle = 'constant'
+drich.Wells.pumping_well.Method = "Standard"
+drich.Wells.pumping_well.Cycle = "constant"
 drich.Wells.pumping_well.alltime.Pressure.Value = 0.5
 drich.Wells.pumping_well.alltime.Saturation.water.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.BCPressure.PatchNames = 'left right front back bottom top'
+drich.BCPressure.PatchNames = "left right front back bottom top"
 
-drich.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-drich.Patch.left.BCPressure.Cycle = 'constant'
-drich.Patch.left.BCPressure.RefGeom = 'domain'
-drich.Patch.left.BCPressure.RefPatch = 'bottom'
+drich.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+drich.Patch.left.BCPressure.Cycle = "constant"
+drich.Patch.left.BCPressure.RefGeom = "domain"
+drich.Patch.left.BCPressure.RefPatch = "bottom"
 drich.Patch.left.BCPressure.alltime.Value = 5.0
 
-drich.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-drich.Patch.right.BCPressure.Cycle = 'constant'
-drich.Patch.right.BCPressure.RefGeom = 'domain'
-drich.Patch.right.BCPressure.RefPatch = 'bottom'
+drich.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+drich.Patch.right.BCPressure.Cycle = "constant"
+drich.Patch.right.BCPressure.RefGeom = "domain"
+drich.Patch.right.BCPressure.RefPatch = "bottom"
 drich.Patch.right.BCPressure.alltime.Value = 5.0
 
-drich.Patch.front.BCPressure.Type = 'FluxConst'
-drich.Patch.front.BCPressure.Cycle = 'constant'
+drich.Patch.front.BCPressure.Type = "FluxConst"
+drich.Patch.front.BCPressure.Cycle = "constant"
 drich.Patch.front.BCPressure.alltime.Value = 0.0
 
-drich.Patch.back.BCPressure.Type = 'FluxConst'
-drich.Patch.back.BCPressure.Cycle = 'constant'
+drich.Patch.back.BCPressure.Type = "FluxConst"
+drich.Patch.back.BCPressure.Cycle = "constant"
 drich.Patch.back.BCPressure.alltime.Value = 0.0
 
-drich.Patch.bottom.BCPressure.Type = 'FluxConst'
-drich.Patch.bottom.BCPressure.Cycle = 'constant'
+drich.Patch.bottom.BCPressure.Type = "FluxConst"
+drich.Patch.bottom.BCPressure.Cycle = "constant"
 drich.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-drich.Patch.top.BCPressure.Type = 'FluxConst'
-drich.Patch.top.BCPressure.Cycle = 'constant'
+drich.Patch.top.BCPressure.Type = "FluxConst"
+drich.Patch.top.BCPressure.Cycle = "constant"
 drich.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.TopoSlopesX.Type = 'Constant'
-drich.TopoSlopesX.GeomNames = 'domain'
+drich.TopoSlopesX.Type = "Constant"
+drich.TopoSlopesX.GeomNames = "domain"
 
 drich.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.TopoSlopesY.Type = 'Constant'
-drich.TopoSlopesY.GeomNames = 'domain'
+drich.TopoSlopesY.Type = "Constant"
+drich.TopoSlopesY.GeomNames = "domain"
 drich.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-drich.Mannings.Type = 'Constant'
-drich.Mannings.GeomNames = 'domain'
-drich.Mannings.Geom.domain.Value = 0.
+drich.Mannings.Type = "Constant"
+drich.Mannings.GeomNames = "domain"
+drich.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.ICPressure.Type = 'HydroStaticPatch'
-drich.ICPressure.GeomNames = 'domain'
+drich.ICPressure.Type = "HydroStaticPatch"
+drich.ICPressure.GeomNames = "domain"
 drich.Geom.domain.ICPressure.Value = 5.0
-drich.Geom.domain.ICPressure.RefGeom = 'domain'
-drich.Geom.domain.ICPressure.RefPatch = 'bottom'
+drich.Geom.domain.ICPressure.RefGeom = "domain"
+drich.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.PhaseSources.water.Type = 'Constant'
-drich.PhaseSources.water.GeomNames = 'background'
+drich.PhaseSources.water.Type = "Constant"
+drich.PhaseSources.water.GeomNames = "background"
 drich.PhaseSources.water.Geom.background.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.KnownSolution = 'NoKnownSolution'
+drich.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Solver = 'Richards'
+drich.Solver = "Richards"
 drich.Solver.MaxIter = 5
 
 drich.Solver.Nonlinear.MaxIter = 10
 drich.Solver.Nonlinear.ResidualTol = 1e-9
-drich.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+drich.Solver.Nonlinear.EtaChoice = "EtaConstant"
 drich.Solver.Nonlinear.EtaValue = 1e-5
 drich.Solver.Nonlinear.UseJacobian = True
 drich.Solver.Nonlinear.DerivativeEpsilon = 1e-2
 
 drich.Solver.Linear.KrylovDimension = 10
 
-drich.Solver.Linear.Preconditioner = 'MGSemi'
+drich.Solver.Linear.Preconditioner = "MGSemi"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/drich_w')
-mkdir(dir_name)
-drich.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+new_output_dir_name = get_absolute_path("test_output/" + run_name)
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+drich.run(working_directory=new_output_dir_name)
+
+passed = True
+
+filename = f"/{run_name}.out.perm_x.pfb"
+if not pf_test_file(
+    new_output_dir_name + filename,
+    correct_output_dir_name + filename,
+    "Max difference in perm_x",
+):
+    passed = False
+filename = f"/{run_name}.out.perm_y.pfb"
+if not pf_test_file(
+    new_output_dir_name + filename,
+    correct_output_dir_name + filename,
+    "Max difference in perm_y",
+):
+    passed = False
+filename = f"/{run_name}.out.perm_z.pfb"
+if not pf_test_file(
+    new_output_dir_name + filename,
+    correct_output_dir_name + filename,
+    "Max difference in perm_z",
+):
+    passed = False
+
+timesteps = ["00000", "00001", "00002", "00003", "00004", "00005"]
+for i in timesteps:
+    filename = f"/{run_name}.out.press.{i}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {i}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{i}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {i}",
+    ):
+        passed = False
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/default_richards_with_silo.py b/test/python/default_richards_with_silo.py
index b096818f8..3b08f3794 100644
--- a/test/python/default_richards_with_silo.py
+++ b/test/python/default_richards_with_silo.py
@@ -1,15 +1,15 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  This runs the basic default_richards test case.
 #  This run, as written in this input file, should take
 #  3 nonlinear iterations.
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
 
 drich = Run("default_richards_with_silo", __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.FileVersion = 4
 
@@ -17,9 +17,9 @@
 drich.Process.Topology.Q = 1
 drich.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.ComputationalGrid.Lower.X = -10.0
 drich.ComputationalGrid.Lower.Y = 10.0
@@ -33,22 +33,24 @@
 drich.ComputationalGrid.NY = 10
 drich.ComputationalGrid.NZ = 8
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.Names = 'domain_input background_input source_region_input concen_region_input'
+drich.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.domain_input.InputType = 'Box'
-drich.GeomInput.domain_input.GeomName = 'domain'
+drich.GeomInput.domain_input.InputType = "Box"
+drich.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Geom.domain.Lower.X = -10.0
 drich.Geom.domain.Lower.Y = 10.0
@@ -58,18 +60,18 @@
 drich.Geom.domain.Upper.Y = 170.0
 drich.Geom.domain.Upper.Z = 9.0
 
-drich.Geom.domain.Patches = 'left right front back bottom top'
+drich.Geom.domain.Patches = "left right front back bottom top"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.background_input.InputType = 'Box'
-drich.GeomInput.background_input.GeomName = 'background'
+drich.GeomInput.background_input.InputType = "Box"
+drich.GeomInput.background_input.GeomName = "background"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Geom.background.Lower.X = -99999999.0
 drich.Geom.background.Lower.Y = -99999999.0
@@ -79,16 +81,16 @@
 drich.Geom.background.Upper.Y = 99999999.0
 drich.Geom.background.Upper.Z = 99999999.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.source_region_input.InputType = 'Box'
-drich.GeomInput.source_region_input.GeomName = 'source_region'
+drich.GeomInput.source_region_input.InputType = "Box"
+drich.GeomInput.source_region_input.GeomName = "source_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Geom.source_region.Lower.X = 65.56
 drich.Geom.source_region.Lower.Y = 79.34
@@ -98,16 +100,16 @@
 drich.Geom.source_region.Upper.Y = 89.99
 drich.Geom.source_region.Upper.Z = 5.5
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.concen_region_input.InputType = 'Box'
-drich.GeomInput.concen_region_input.GeomName = 'concen_region'
+drich.GeomInput.concen_region_input.InputType = "Box"
+drich.GeomInput.concen_region_input.GeomName = "concen_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Geom.concen_region.Lower.X = 60.0
 drich.Geom.concen_region.Lower.Y = 80.0
@@ -117,231 +119,231 @@
 drich.Geom.concen_region.Upper.Y = 100.0
 drich.Geom.concen_region.Upper.Z = 6.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Geom.Perm.Names = 'background'
+drich.Geom.Perm.Names = "background"
 
-drich.Geom.background.Perm.Type = 'Constant'
+drich.Geom.background.Perm.Type = "Constant"
 drich.Geom.background.Perm.Value = 4.0
 
-drich.Perm.TensorType = 'TensorByGeom'
+drich.Perm.TensorType = "TensorByGeom"
 
-drich.Geom.Perm.TensorByGeom.Names = 'background'
+drich.Geom.Perm.TensorByGeom.Names = "background"
 
 drich.Geom.background.Perm.TensorValX = 1.0
 drich.Geom.background.Perm.TensorValY = 1.0
 drich.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.SpecificStorage.Type = 'Constant'
-drich.SpecificStorage.GeomNames = 'domain'
+drich.SpecificStorage.Type = "Constant"
+drich.SpecificStorage.GeomNames = "domain"
 drich.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Phase.Names = 'water'
+drich.Phase.Names = "water"
 
-drich.Phase.water.Density.Type = 'Constant'
+drich.Phase.water.Density.Type = "Constant"
 drich.Phase.water.Density.Value = 1.0
 
-drich.Phase.water.Viscosity.Type = 'Constant'
+drich.Phase.water.Viscosity.Type = "Constant"
 drich.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Contaminants.Names = ''
+drich.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Geom.Retardation.GeomNames = ''
+drich.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.TimingInfo.BaseUnit = 1.0
 drich.TimingInfo.StartCount = 0
 drich.TimingInfo.StartTime = 0.0
 drich.TimingInfo.StopTime = 0.010
 drich.TimingInfo.DumpInterval = -1
-drich.TimeStep.Type = 'Constant'
+drich.TimeStep.Type = "Constant"
 drich.TimeStep.Value = 0.001
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Geom.Porosity.GeomNames = 'background'
-drich.Geom.background.Porosity.Type = 'Constant'
+drich.Geom.Porosity.GeomNames = "background"
+drich.Geom.background.Porosity.Type = "Constant"
 drich.Geom.background.Porosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Domain.GeomName = 'domain'
+drich.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Phase.RelPerm.Type = 'VanGenuchten'
-drich.Phase.RelPerm.GeomNames = 'domain'
+drich.Phase.RelPerm.Type = "VanGenuchten"
+drich.Phase.RelPerm.GeomNames = "domain"
 drich.Geom.domain.RelPerm.Alpha = 0.005
 drich.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.Phase.Saturation.Type = 'VanGenuchten'
-drich.Phase.Saturation.GeomNames = 'domain'
+drich.Phase.Saturation.Type = "VanGenuchten"
+drich.Phase.Saturation.GeomNames = "domain"
 drich.Geom.domain.Saturation.Alpha = 0.005
 drich.Geom.domain.Saturation.N = 2.0
 drich.Geom.domain.Saturation.SRes = 0.2
 drich.Geom.domain.Saturation.SSat = 0.99
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Wells.Names = ''
+drich.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Cycle.Names = 'constant'
-drich.Cycle.constant.Names = 'alltime'
+drich.Cycle.Names = "constant"
+drich.Cycle.constant.Names = "alltime"
 drich.Cycle.constant.alltime.Length = 1
 drich.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.BCPressure.PatchNames = 'left right front back bottom top'
+drich.BCPressure.PatchNames = "left right front back bottom top"
 
-drich.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-drich.Patch.left.BCPressure.Cycle = 'constant'
-drich.Patch.left.BCPressure.RefGeom = 'domain'
-drich.Patch.left.BCPressure.RefPatch = 'bottom'
+drich.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+drich.Patch.left.BCPressure.Cycle = "constant"
+drich.Patch.left.BCPressure.RefGeom = "domain"
+drich.Patch.left.BCPressure.RefPatch = "bottom"
 drich.Patch.left.BCPressure.alltime.Value = 5.0
 
-drich.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-drich.Patch.right.BCPressure.Cycle = 'constant'
-drich.Patch.right.BCPressure.RefGeom = 'domain'
-drich.Patch.right.BCPressure.RefPatch = 'bottom'
+drich.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+drich.Patch.right.BCPressure.Cycle = "constant"
+drich.Patch.right.BCPressure.RefGeom = "domain"
+drich.Patch.right.BCPressure.RefPatch = "bottom"
 drich.Patch.right.BCPressure.alltime.Value = 3.0
 
-drich.Patch.front.BCPressure.Type = 'FluxConst'
-drich.Patch.front.BCPressure.Cycle = 'constant'
+drich.Patch.front.BCPressure.Type = "FluxConst"
+drich.Patch.front.BCPressure.Cycle = "constant"
 drich.Patch.front.BCPressure.alltime.Value = 0.0
 
-drich.Patch.back.BCPressure.Type = 'FluxConst'
-drich.Patch.back.BCPressure.Cycle = 'constant'
+drich.Patch.back.BCPressure.Type = "FluxConst"
+drich.Patch.back.BCPressure.Cycle = "constant"
 drich.Patch.back.BCPressure.alltime.Value = 0.0
 
-drich.Patch.bottom.BCPressure.Type = 'FluxConst'
-drich.Patch.bottom.BCPressure.Cycle = 'constant'
+drich.Patch.bottom.BCPressure.Type = "FluxConst"
+drich.Patch.bottom.BCPressure.Cycle = "constant"
 drich.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-drich.Patch.top.BCPressure.Type = 'FluxConst'
-drich.Patch.top.BCPressure.Cycle = 'constant'
+drich.Patch.top.BCPressure.Type = "FluxConst"
+drich.Patch.top.BCPressure.Cycle = "constant"
 drich.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.TopoSlopesX.Type = 'Constant'
-drich.TopoSlopesX.GeomNames = 'domain'
+drich.TopoSlopesX.Type = "Constant"
+drich.TopoSlopesX.GeomNames = "domain"
 drich.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.TopoSlopesY.Type = 'Constant'
-drich.TopoSlopesY.GeomNames = 'domain'
+drich.TopoSlopesY.Type = "Constant"
+drich.TopoSlopesY.GeomNames = "domain"
 drich.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-drich.Mannings.Type = 'Constant'
-drich.Mannings.GeomNames = 'domain'
-drich.Mannings.Geom.domain.Value = 0.
+drich.Mannings.Type = "Constant"
+drich.Mannings.GeomNames = "domain"
+drich.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.ICPressure.Type = 'HydroStaticPatch'
-drich.ICPressure.GeomNames = 'domain'
+drich.ICPressure.Type = "HydroStaticPatch"
+drich.ICPressure.GeomNames = "domain"
 drich.Geom.domain.ICPressure.Value = 3.0
-drich.Geom.domain.ICPressure.RefGeom = 'domain'
-drich.Geom.domain.ICPressure.RefPatch = 'bottom'
+drich.Geom.domain.ICPressure.RefGeom = "domain"
+drich.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.PhaseSources.water.Type = 'Constant'
-drich.PhaseSources.water.GeomNames = 'background'
+drich.PhaseSources.water.Type = "Constant"
+drich.PhaseSources.water.GeomNames = "background"
 drich.PhaseSources.water.Geom.background.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.KnownSolution = 'NoKnownSolution'
+drich.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Solver = 'Richards'
+drich.Solver = "Richards"
 drich.Solver.MaxIter = 5
 
 drich.Solver.Nonlinear.MaxIter = 10
 drich.Solver.Nonlinear.ResidualTol = 1e-9
-drich.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+drich.Solver.Nonlinear.EtaChoice = "EtaConstant"
 drich.Solver.Nonlinear.EtaValue = 1e-5
 drich.Solver.Nonlinear.UseJacobian = True
 drich.Solver.Nonlinear.DerivativeEpsilon = 1e-2
 
 drich.Solver.Linear.KrylovDimension = 10
 
-drich.Solver.Linear.Preconditioner = 'PFMGOctree'
+drich.Solver.Linear.Preconditioner = "PFMGOctree"
 
 drich.Solver.WriteSiloSubsurfData = True
 drich.Solver.WriteSiloPressure = True
 drich.Solver.WriteSiloSaturation = True
 drich.Solver.WriteSiloConcentration = True
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/drich_s')
+dir_name = get_absolute_path("test_output/drich_s")
 mkdir(dir_name)
 drich.run(working_directory=dir_name)
diff --git a/test/python/default_single.py b/test/python/default_single.py
index bb31928fd..5fe6e8200 100644
--- a/test/python/default_single.py
+++ b/test/python/default_single.py
@@ -1,340 +1,418 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # default_single test run
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file, pf_test_file_with_abs
+import sys
 
-dsingle = Run("dsingle", __file__)
+run_name = "default_single"
+default_single = Run(run_name, __file__)
 
-#-----------------------------------------------------------------------------
+correct_output_dir_name = get_absolute_path("../correct_output")
 
-dsingle.FileVersion = 4
+# -----------------------------------------------------------------------------
 
-#-----------------------------------------------------------------------------
+default_single.FileVersion = 4
+
+# -----------------------------------------------------------------------------
 # Process Topology
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Process.Topology.P = 1
-dsingle.Process.Topology.Q = 1
-dsingle.Process.Topology.R = 1
+default_single.Process.Topology.P = 1
+default_single.Process.Topology.Q = 1
+default_single.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.ComputationalGrid.Lower.X = -10.0
-dsingle.ComputationalGrid.Lower.Y = 10.0
-dsingle.ComputationalGrid.Lower.Z = 1.0
+default_single.ComputationalGrid.Lower.X = -10.0
+default_single.ComputationalGrid.Lower.Y = 10.0
+default_single.ComputationalGrid.Lower.Z = 1.0
 
-dsingle.ComputationalGrid.DX = 8.8888888888888893
-dsingle.ComputationalGrid.DY = 10.666666666666666
-dsingle.ComputationalGrid.DZ = 1.0
+default_single.ComputationalGrid.DX = 8.8888888888888893
+default_single.ComputationalGrid.DY = 10.666666666666666
+default_single.ComputationalGrid.DZ = 1.0
 
-dsingle.ComputationalGrid.NX = 18
-dsingle.ComputationalGrid.NY = 15
-dsingle.ComputationalGrid.NZ = 8
+default_single.ComputationalGrid.NX = 18
+default_single.ComputationalGrid.NY = 15
+default_single.ComputationalGrid.NZ = 8
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # The Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.GeomInput.Names = 'domain_input background_input source_region_input concen_region_input'
+default_single.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.GeomInput.domain_input.InputType = 'Box'
-dsingle.GeomInput.domain_input.GeomName = 'domain'
+default_single.GeomInput.domain_input.InputType = "Box"
+default_single.GeomInput.domain_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Geom.domain.Lower.X = -10.0
-dsingle.Geom.domain.Lower.Y = 10.0
-dsingle.Geom.domain.Lower.Z = 1.0
+default_single.Geom.domain.Lower.X = -10.0
+default_single.Geom.domain.Lower.Y = 10.0
+default_single.Geom.domain.Lower.Z = 1.0
 
-dsingle.Geom.domain.Upper.X = 150.0
-dsingle.Geom.domain.Upper.Y = 170.0
-dsingle.Geom.domain.Upper.Z = 9.0
+default_single.Geom.domain.Upper.X = 150.0
+default_single.Geom.domain.Upper.Y = 170.0
+default_single.Geom.domain.Upper.Z = 9.0
 
-dsingle.Geom.domain.Patches = 'left right front back bottom top'
+default_single.Geom.domain.Patches = "left right front back bottom top"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Background Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.GeomInput.background_input.InputType = 'Box'
-dsingle.GeomInput.background_input.GeomName = 'background'
+default_single.GeomInput.background_input.InputType = "Box"
+default_single.GeomInput.background_input.GeomName = "background"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Background Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Geom.background.Lower.X = -99999999.0
-dsingle.Geom.background.Lower.Y = -99999999.0
-dsingle.Geom.background.Lower.Z = -99999999.0
+default_single.Geom.background.Lower.X = -99999999.0
+default_single.Geom.background.Lower.Y = -99999999.0
+default_single.Geom.background.Lower.Z = -99999999.0
 
-dsingle.Geom.background.Upper.X = 99999999.0
-dsingle.Geom.background.Upper.Y = 99999999.0
-dsingle.Geom.background.Upper.Z = 99999999.0
+default_single.Geom.background.Upper.X = 99999999.0
+default_single.Geom.background.Upper.Y = 99999999.0
+default_single.Geom.background.Upper.Z = 99999999.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Source_Region Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.GeomInput.source_region_input.InputType = 'Box'
-dsingle.GeomInput.source_region_input.GeomName = 'source_region'
+default_single.GeomInput.source_region_input.InputType = "Box"
+default_single.GeomInput.source_region_input.GeomName = "source_region"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Source_Region Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Geom.source_region.Lower.X = 65.56
-dsingle.Geom.source_region.Lower.Y = 79.34
-dsingle.Geom.source_region.Lower.Z = 4.5
+default_single.Geom.source_region.Lower.X = 65.56
+default_single.Geom.source_region.Lower.Y = 79.34
+default_single.Geom.source_region.Lower.Z = 4.5
 
-dsingle.Geom.source_region.Upper.X = 74.44
-dsingle.Geom.source_region.Upper.Y = 89.99
-dsingle.Geom.source_region.Upper.Z = 5.5
+default_single.Geom.source_region.Upper.X = 74.44
+default_single.Geom.source_region.Upper.Y = 89.99
+default_single.Geom.source_region.Upper.Z = 5.5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Concen_Region Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.GeomInput.concen_region_input.InputType = 'Box'
-dsingle.GeomInput.concen_region_input.GeomName = 'concen_region'
+default_single.GeomInput.concen_region_input.InputType = "Box"
+default_single.GeomInput.concen_region_input.GeomName = "concen_region"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Concen_Region Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Geom.concen_region.Lower.X = 60.0
-dsingle.Geom.concen_region.Lower.Y = 80.0
-dsingle.Geom.concen_region.Lower.Z = 4.0
+default_single.Geom.concen_region.Lower.X = 60.0
+default_single.Geom.concen_region.Lower.Y = 80.0
+default_single.Geom.concen_region.Lower.Z = 4.0
 
-dsingle.Geom.concen_region.Upper.X = 80.0
-dsingle.Geom.concen_region.Upper.Y = 100.0
-dsingle.Geom.concen_region.Upper.Z = 6.0
+default_single.Geom.concen_region.Upper.X = 80.0
+default_single.Geom.concen_region.Upper.Y = 100.0
+default_single.Geom.concen_region.Upper.Z = 6.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Geom.Perm.Names = 'background'
+default_single.Geom.Perm.Names = "background"
 
-dsingle.Geom.background.Perm.Type = 'Constant'
-dsingle.Geom.background.Perm.Value = 4.0
+default_single.Geom.background.Perm.Type = "Constant"
+default_single.Geom.background.Perm.Value = 4.0
 
-dsingle.Perm.TensorType = 'TensorByGeom'
+default_single.Perm.TensorType = "TensorByGeom"
 
-dsingle.Geom.Perm.TensorByGeom.Names = 'background'
+default_single.Geom.Perm.TensorByGeom.Names = "background"
 
-dsingle.Geom.background.Perm.TensorValX = 1.0
-dsingle.Geom.background.Perm.TensorValY = 1.0
-dsingle.Geom.background.Perm.TensorValZ = 1.0
+default_single.Geom.background.Perm.TensorValX = 1.0
+default_single.Geom.background.Perm.TensorValY = 1.0
+default_single.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 # specific storage does not figure into the impes (fully sat) case but we still
 # need a key for it
-dsingle.SpecificStorage.Type = 'Constant'
-dsingle.SpecificStorage.GeomNames = ''
-dsingle.Geom.domain.SpecificStorage.Value = 1.0e-4
+default_single.SpecificStorage.Type = "Constant"
+default_single.SpecificStorage.GeomNames = ""
+default_single.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Phase.Names = 'water'
+default_single.Phase.Names = "water"
 
-dsingle.Phase.water.Density.Type = 'Constant'
-dsingle.Phase.water.Density.Value = 1.0
+default_single.Phase.water.Density.Type = "Constant"
+default_single.Phase.water.Density.Value = 1.0
 
-dsingle.Phase.water.Viscosity.Type = 'Constant'
-dsingle.Phase.water.Viscosity.Value = 1.0
+default_single.Phase.water.Viscosity.Type = "Constant"
+default_single.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Contaminants.Names = 'tce'
-dsingle.Contaminants.tce.Degradation.Value = 0.0
+default_single.Contaminants.Names = "tce"
+default_single.Contaminants.tce.Degradation.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Gravity = 1.0
+default_single.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.TimingInfo.BaseUnit = 1.0
-dsingle.TimingInfo.StartCount = 0
-dsingle.TimingInfo.StartTime = 0.0
-dsingle.TimingInfo.StopTime = 1000.0
-dsingle.TimingInfo.DumpInterval = -1
+default_single.TimingInfo.BaseUnit = 1.0
+default_single.TimingInfo.StartCount = 0
+default_single.TimingInfo.StartTime = 0.0
+default_single.TimingInfo.StopTime = 1000.0
+default_single.TimingInfo.DumpInterval = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Geom.Porosity.GeomNames = 'background'
-dsingle.Geom.background.Porosity.Type = 'Constant'
-dsingle.Geom.background.Porosity.Value = 1.0
+default_single.Geom.Porosity.GeomNames = "background"
+default_single.Geom.background.Porosity.Type = "Constant"
+default_single.Geom.background.Porosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Domain.GeomName = 'domain'
+default_single.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mobility
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Phase.water.Mobility.Type = 'Constant'
-dsingle.Phase.water.Mobility.Value = 1.0
+default_single.Phase.water.Mobility.Type = "Constant"
+default_single.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Geom.Retardation.GeomNames = 'background'
-dsingle.Geom.background.tce.Retardation.Type = 'Linear'
-dsingle.Geom.background.tce.Retardation.Rate = 0.0
+default_single.Geom.Retardation.GeomNames = "background"
+default_single.Geom.background.tce.Retardation.Type = "Linear"
+default_single.Geom.background.tce.Retardation.Rate = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Cycle.Names = 'constant'
-dsingle.Cycle.constant.Names = 'alltime'
-dsingle.Cycle.constant.alltime.Length = 1
-dsingle.Cycle.constant.Repeat = -1
+default_single.Cycle.Names = "constant"
+default_single.Cycle.constant.Names = "alltime"
+default_single.Cycle.constant.alltime.Length = 1
+default_single.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Wells.Names = 'snoopy'
+default_single.Wells.Names = "snoopy"
 
-dsingle.Wells.snoopy.InputType = 'Recirc'
+default_single.Wells.snoopy.InputType = "Recirc"
 
-dsingle.Wells.snoopy.Cycle = 'constant'
+default_single.Wells.snoopy.Cycle = "constant"
 
-dsingle.Wells.snoopy.ExtractionType = 'Flux'
-dsingle.Wells.snoopy.InjectionType = 'Flux'
+default_single.Wells.snoopy.ExtractionType = "Flux"
+default_single.Wells.snoopy.InjectionType = "Flux"
 
-dsingle.Wells.snoopy.X = 71.0
-dsingle.Wells.snoopy.Y = 90.0
-dsingle.Wells.snoopy.ExtractionZLower = 5.0
-dsingle.Wells.snoopy.ExtractionZUpper = 5.0
-dsingle.Wells.snoopy.InjectionZLower = 2.0
-dsingle.Wells.snoopy.InjectionZUpper = 2.0
+default_single.Wells.snoopy.X = 71.0
+default_single.Wells.snoopy.Y = 90.0
+default_single.Wells.snoopy.ExtractionZLower = 5.0
+default_single.Wells.snoopy.ExtractionZUpper = 5.0
+default_single.Wells.snoopy.InjectionZLower = 2.0
+default_single.Wells.snoopy.InjectionZUpper = 2.0
 
-dsingle.Wells.snoopy.ExtractionMethod = 'Standard'
-dsingle.Wells.snoopy.InjectionMethod = 'Standard'
+default_single.Wells.snoopy.ExtractionMethod = "Standard"
+default_single.Wells.snoopy.InjectionMethod = "Standard"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Assigning well with newly assigned interval name
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Wells.snoopy.alltime.Extraction.Flux.water.Value = 5.0
-dsingle.Wells.snoopy.alltime.Injection.Flux.water.Value = 7.5
-dsingle.Wells.snoopy.alltime.Injection.Concentration.water.tce.Fraction = 0.1
+default_single.Wells.snoopy.alltime.Extraction.Flux.water.Value = 5.0
+default_single.Wells.snoopy.alltime.Injection.Flux.water.Value = 7.5
+default_single.Wells.snoopy.alltime.Injection.Concentration.water.tce.Fraction = 0.1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.BCPressure.PatchNames = 'left right front back bottom top'
+default_single.BCPressure.PatchNames = "left right front back bottom top"
 
-dsingle.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-dsingle.Patch.left.BCPressure.Cycle = 'constant'
-dsingle.Patch.left.BCPressure.RefGeom = 'domain'
-dsingle.Patch.left.BCPressure.RefPatch = 'bottom'
-dsingle.Patch.left.BCPressure.alltime.Value = 14.0
+default_single.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+default_single.Patch.left.BCPressure.Cycle = "constant"
+default_single.Patch.left.BCPressure.RefGeom = "domain"
+default_single.Patch.left.BCPressure.RefPatch = "bottom"
+default_single.Patch.left.BCPressure.alltime.Value = 14.0
 
-dsingle.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-dsingle.Patch.right.BCPressure.Cycle = 'constant'
-dsingle.Patch.right.BCPressure.RefGeom = 'domain'
-dsingle.Patch.right.BCPressure.RefPatch = 'bottom'
-dsingle.Patch.right.BCPressure.alltime.Value = 9.0
+default_single.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+default_single.Patch.right.BCPressure.Cycle = "constant"
+default_single.Patch.right.BCPressure.RefGeom = "domain"
+default_single.Patch.right.BCPressure.RefPatch = "bottom"
+default_single.Patch.right.BCPressure.alltime.Value = 9.0
 
-dsingle.Patch.front.BCPressure.Type = 'FluxConst'
-dsingle.Patch.front.BCPressure.Cycle = 'constant'
-dsingle.Patch.front.BCPressure.alltime.Value = 0.0
+default_single.Patch.front.BCPressure.Type = "FluxConst"
+default_single.Patch.front.BCPressure.Cycle = "constant"
+default_single.Patch.front.BCPressure.alltime.Value = 0.0
 
-dsingle.Patch.back.BCPressure.Type = 'FluxConst'
-dsingle.Patch.back.BCPressure.Cycle = 'constant'
-dsingle.Patch.back.BCPressure.alltime.Value = 0.0
+default_single.Patch.back.BCPressure.Type = "FluxConst"
+default_single.Patch.back.BCPressure.Cycle = "constant"
+default_single.Patch.back.BCPressure.alltime.Value = 0.0
 
-dsingle.Patch.bottom.BCPressure.Type = 'FluxConst'
-dsingle.Patch.bottom.BCPressure.Cycle = 'constant'
-dsingle.Patch.bottom.BCPressure.alltime.Value = 0.0
+default_single.Patch.bottom.BCPressure.Type = "FluxConst"
+default_single.Patch.bottom.BCPressure.Cycle = "constant"
+default_single.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-dsingle.Patch.top.BCPressure.Type = 'FluxConst'
-dsingle.Patch.top.BCPressure.Cycle = 'constant'
-dsingle.Patch.top.BCPressure.alltime.Value = 0.0
+default_single.Patch.top.BCPressure.Type = "FluxConst"
+default_single.Patch.top.BCPressure.Cycle = "constant"
+default_single.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # topo slopes do not figure into the impes (fully sat) case but we still
 # need keys for them
 
-dsingle.TopoSlopesX.Type = 'Constant'
-dsingle.TopoSlopesX.GeomNames = ''
-dsingle.TopoSlopesX.Geom.domain.Value = 0.0
+default_single.TopoSlopesX.Type = "Constant"
+default_single.TopoSlopesX.GeomNames = ""
+default_single.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dsingle.TopoSlopesY.Type = 'Constant'
-dsingle.TopoSlopesY.GeomNames = ''
-dsingle.TopoSlopesY.Geom.domain.Value = 0.0
+default_single.TopoSlopesY.Type = "Constant"
+default_single.TopoSlopesY.GeomNames = ""
+default_single.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # mannings roughnesses do not figure into the impes (fully sat) case but we still
 # need a key for them
 
-dsingle.Mannings.Type = 'Constant'
-dsingle.Mannings.GeomNames = ''
-dsingle.Mannings.Geom.domain.Value = 0.0
+default_single.Mannings.Type = "Constant"
+default_single.Mannings.GeomNames = ""
+default_single.Mannings.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+
+default_single.PhaseSources.water.Type = "Constant"
+default_single.PhaseSources.water.GeomNames = "background"
+default_single.PhaseSources.water.Geom.background.Value = 0.0
 
-dsingle.PhaseSources.water.Type = 'Constant'
-dsingle.PhaseSources.water.GeomNames = 'background'
-dsingle.PhaseSources.water.Geom.background.Value = 0.0
+default_single.PhaseConcen.water.tce.Type = "Constant"
+default_single.PhaseConcen.water.tce.GeomNames = "concen_region"
+default_single.PhaseConcen.water.tce.Geom.concen_region.Value = 0.8
 
-dsingle.PhaseConcen.water.tce.Type = 'Constant'
-dsingle.PhaseConcen.water.tce.GeomNames = 'concen_region'
-dsingle.PhaseConcen.water.tce.Geom.concen_region.Value = 0.8
+default_single.Solver.PrintVelocities = True
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # The Solver Impes MaxIter default value changed so to get previous
 # results we need to set it back to what it was
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Solver.MaxIter = 5
+default_single.Solver.MaxIter = 5
+default_single.Solver.AbsTol = 1e-25
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/dsingle')
-mkdir(dir_name)
-dsingle.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/default_single")
+mkdir(new_output_dir_name)
+default_single.run(working_directory=new_output_dir_name)
+
+# -----------------------------------------------------------------------------
+# Check results for regression
+# -----------------------------------------------------------------------------
+
+passed = True
+
+i = 0
+timestep = str(i).rjust(5, "0")
+
+sig_digits = 4
+abs_value = 1e-12
+
+test_files = ["press"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {filename}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {filename}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+test_files = ["phasex", "phasey", "phasez"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.0.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {filename}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+
+test_files = ["concen.0.00"]
+for i in range(0, 5):
+    timestep = str(i).rjust(5, "0")
+    for test_file in test_files:
+        filename = f"/{run_name}.out.{test_file}.{timestep}.pfsb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in {test_file}",
+            sig_digits,
+        ):
+            passed = False
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/default_single_read_porosity.py b/test/python/default_single_read_porosity.py
new file mode 100644
index 000000000..ef2756ed2
--- /dev/null
+++ b/test/python/default_single_read_porosity.py
@@ -0,0 +1,425 @@
+# -----------------------------------------------------------------------------
+# default_single test run
+# -----------------------------------------------------------------------------
+
+from parflow import Run
+from parflow.tools.fs import cp, mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file, pf_test_file_with_abs
+import sys
+
+run_name = "default_single"
+default_single = Run(run_name, __file__)
+
+correct_output_dir_name = get_absolute_path("../correct_output")
+
+# -----------------------------------------------------------------------------
+
+default_single.FileVersion = 4
+
+# -----------------------------------------------------------------------------
+# Process Topology
+# -----------------------------------------------------------------------------
+
+default_single.Process.Topology.P = 1
+default_single.Process.Topology.Q = 1
+default_single.Process.Topology.R = 1
+
+# -----------------------------------------------------------------------------
+# Computational Grid
+# -----------------------------------------------------------------------------
+
+default_single.ComputationalGrid.Lower.X = -10.0
+default_single.ComputationalGrid.Lower.Y = 10.0
+default_single.ComputationalGrid.Lower.Z = 1.0
+
+default_single.ComputationalGrid.DX = 8.8888888888888893
+default_single.ComputationalGrid.DY = 10.666666666666666
+default_single.ComputationalGrid.DZ = 1.0
+
+default_single.ComputationalGrid.NX = 18
+default_single.ComputationalGrid.NY = 15
+default_single.ComputationalGrid.NZ = 8
+
+# -----------------------------------------------------------------------------
+# The Names of the GeomInputs
+# -----------------------------------------------------------------------------
+
+default_single.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
+
+# -----------------------------------------------------------------------------
+# Domain Geometry Input
+# -----------------------------------------------------------------------------
+
+default_single.GeomInput.domain_input.InputType = "Box"
+default_single.GeomInput.domain_input.GeomName = "domain"
+
+# -----------------------------------------------------------------------------
+# Domain Geometry
+# -----------------------------------------------------------------------------
+
+default_single.Geom.domain.Lower.X = -10.0
+default_single.Geom.domain.Lower.Y = 10.0
+default_single.Geom.domain.Lower.Z = 1.0
+
+default_single.Geom.domain.Upper.X = 150.0
+default_single.Geom.domain.Upper.Y = 170.0
+default_single.Geom.domain.Upper.Z = 9.0
+
+default_single.Geom.domain.Patches = "left right front back bottom top"
+
+# -----------------------------------------------------------------------------
+# Background Geometry Input
+# -----------------------------------------------------------------------------
+
+default_single.GeomInput.background_input.InputType = "Box"
+default_single.GeomInput.background_input.GeomName = "background"
+
+# -----------------------------------------------------------------------------
+# Background Geometry
+# -----------------------------------------------------------------------------
+
+default_single.Geom.background.Lower.X = -99999999.0
+default_single.Geom.background.Lower.Y = -99999999.0
+default_single.Geom.background.Lower.Z = -99999999.0
+
+default_single.Geom.background.Upper.X = 99999999.0
+default_single.Geom.background.Upper.Y = 99999999.0
+default_single.Geom.background.Upper.Z = 99999999.0
+
+# -----------------------------------------------------------------------------
+# Source_Region Geometry Input
+# -----------------------------------------------------------------------------
+
+default_single.GeomInput.source_region_input.InputType = "Box"
+default_single.GeomInput.source_region_input.GeomName = "source_region"
+
+# -----------------------------------------------------------------------------
+# Source_Region Geometry
+# -----------------------------------------------------------------------------
+
+default_single.Geom.source_region.Lower.X = 65.56
+default_single.Geom.source_region.Lower.Y = 79.34
+default_single.Geom.source_region.Lower.Z = 4.5
+
+default_single.Geom.source_region.Upper.X = 74.44
+default_single.Geom.source_region.Upper.Y = 89.99
+default_single.Geom.source_region.Upper.Z = 5.5
+
+# -----------------------------------------------------------------------------
+# Concen_Region Geometry Input
+# -----------------------------------------------------------------------------
+
+default_single.GeomInput.concen_region_input.InputType = "Box"
+default_single.GeomInput.concen_region_input.GeomName = "concen_region"
+
+# -----------------------------------------------------------------------------
+# Concen_Region Geometry
+# -----------------------------------------------------------------------------
+
+default_single.Geom.concen_region.Lower.X = 60.0
+default_single.Geom.concen_region.Lower.Y = 80.0
+default_single.Geom.concen_region.Lower.Z = 4.0
+
+default_single.Geom.concen_region.Upper.X = 80.0
+default_single.Geom.concen_region.Upper.Y = 100.0
+default_single.Geom.concen_region.Upper.Z = 6.0
+
+# -----------------------------------------------------------------------------
+# Perm
+# -----------------------------------------------------------------------------
+
+default_single.Geom.Perm.Names = "background"
+
+default_single.Geom.background.Perm.Type = "Constant"
+default_single.Geom.background.Perm.Value = 4.0
+
+default_single.Perm.TensorType = "TensorByGeom"
+
+default_single.Geom.Perm.TensorByGeom.Names = "background"
+
+default_single.Geom.background.Perm.TensorValX = 1.0
+default_single.Geom.background.Perm.TensorValY = 1.0
+default_single.Geom.background.Perm.TensorValZ = 1.0
+
+# -----------------------------------------------------------------------------
+# Specific Storage
+# -----------------------------------------------------------------------------
+
+# specific storage does not figure into the impes (fully sat) case but we still
+# need a key for it
+default_single.SpecificStorage.Type = "Constant"
+default_single.SpecificStorage.GeomNames = ""
+default_single.Geom.domain.SpecificStorage.Value = 1.0e-4
+
+# -----------------------------------------------------------------------------
+# Phases
+# -----------------------------------------------------------------------------
+
+default_single.Phase.Names = "water"
+
+default_single.Phase.water.Density.Type = "Constant"
+default_single.Phase.water.Density.Value = 1.0
+
+default_single.Phase.water.Viscosity.Type = "Constant"
+default_single.Phase.water.Viscosity.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Contaminants
+# -----------------------------------------------------------------------------
+
+default_single.Contaminants.Names = "tce"
+default_single.Contaminants.tce.Degradation.Value = 0.0
+
+# -----------------------------------------------------------------------------
+# Gravity
+# -----------------------------------------------------------------------------
+
+default_single.Gravity = 1.0
+
+# -----------------------------------------------------------------------------
+# Setup timing info
+# -----------------------------------------------------------------------------
+
+default_single.TimingInfo.BaseUnit = 1.0
+default_single.TimingInfo.StartCount = 0
+default_single.TimingInfo.StartTime = 0.0
+default_single.TimingInfo.StopTime = 1000.0
+default_single.TimingInfo.DumpInterval = -1
+
+# -----------------------------------------------------------------------------
+# Porosity
+# -----------------------------------------------------------------------------
+
+default_single.Geom.Porosity.GeomNames = "background"
+default_single.Geom.background.Porosity.Type = "PFBFile"
+default_single.Geom.background.Porosity.FileName = "default_single_input_porosity.pfb"
+
+# -----------------------------------------------------------------------------
+# Domain
+# -----------------------------------------------------------------------------
+
+default_single.Domain.GeomName = "domain"
+
+# -----------------------------------------------------------------------------
+# Mobility
+# -----------------------------------------------------------------------------
+
+default_single.Phase.water.Mobility.Type = "Constant"
+default_single.Phase.water.Mobility.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Retardation
+# -----------------------------------------------------------------------------
+
+default_single.Geom.Retardation.GeomNames = "background"
+default_single.Geom.background.tce.Retardation.Type = "Linear"
+default_single.Geom.background.tce.Retardation.Rate = 0.0
+
+# -----------------------------------------------------------------------------
+# Time Cycles
+# -----------------------------------------------------------------------------
+
+default_single.Cycle.Names = "constant"
+default_single.Cycle.constant.Names = "alltime"
+default_single.Cycle.constant.alltime.Length = 1
+default_single.Cycle.constant.Repeat = -1
+
+# -----------------------------------------------------------------------------
+# Wells
+# -----------------------------------------------------------------------------
+
+default_single.Wells.Names = "snoopy"
+
+default_single.Wells.snoopy.InputType = "Recirc"
+
+default_single.Wells.snoopy.Cycle = "constant"
+
+default_single.Wells.snoopy.ExtractionType = "Flux"
+default_single.Wells.snoopy.InjectionType = "Flux"
+
+default_single.Wells.snoopy.X = 71.0
+default_single.Wells.snoopy.Y = 90.0
+default_single.Wells.snoopy.ExtractionZLower = 5.0
+default_single.Wells.snoopy.ExtractionZUpper = 5.0
+default_single.Wells.snoopy.InjectionZLower = 2.0
+default_single.Wells.snoopy.InjectionZUpper = 2.0
+
+default_single.Wells.snoopy.ExtractionMethod = "Standard"
+default_single.Wells.snoopy.InjectionMethod = "Standard"
+
+# -----------------------------------------------------------------------------
+# Assigning well with newly assigned interval name
+# -----------------------------------------------------------------------------
+
+default_single.Wells.snoopy.alltime.Extraction.Flux.water.Value = 5.0
+default_single.Wells.snoopy.alltime.Injection.Flux.water.Value = 7.5
+default_single.Wells.snoopy.alltime.Injection.Concentration.water.tce.Fraction = 0.1
+
+# -----------------------------------------------------------------------------
+# Boundary Conditions: Pressure
+# -----------------------------------------------------------------------------
+
+default_single.BCPressure.PatchNames = "left right front back bottom top"
+
+default_single.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+default_single.Patch.left.BCPressure.Cycle = "constant"
+default_single.Patch.left.BCPressure.RefGeom = "domain"
+default_single.Patch.left.BCPressure.RefPatch = "bottom"
+default_single.Patch.left.BCPressure.alltime.Value = 14.0
+
+default_single.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+default_single.Patch.right.BCPressure.Cycle = "constant"
+default_single.Patch.right.BCPressure.RefGeom = "domain"
+default_single.Patch.right.BCPressure.RefPatch = "bottom"
+default_single.Patch.right.BCPressure.alltime.Value = 9.0
+
+default_single.Patch.front.BCPressure.Type = "FluxConst"
+default_single.Patch.front.BCPressure.Cycle = "constant"
+default_single.Patch.front.BCPressure.alltime.Value = 0.0
+
+default_single.Patch.back.BCPressure.Type = "FluxConst"
+default_single.Patch.back.BCPressure.Cycle = "constant"
+default_single.Patch.back.BCPressure.alltime.Value = 0.0
+
+default_single.Patch.bottom.BCPressure.Type = "FluxConst"
+default_single.Patch.bottom.BCPressure.Cycle = "constant"
+default_single.Patch.bottom.BCPressure.alltime.Value = 0.0
+
+default_single.Patch.top.BCPressure.Type = "FluxConst"
+default_single.Patch.top.BCPressure.Cycle = "constant"
+default_single.Patch.top.BCPressure.alltime.Value = 0.0
+
+# ---------------------------------------------------------
+# Topo slopes in x-direction
+# ---------------------------------------------------------
+# topo slopes do not figure into the impes (fully sat) case but we still
+# need keys for them
+
+default_single.TopoSlopesX.Type = "Constant"
+default_single.TopoSlopesX.GeomNames = ""
+default_single.TopoSlopesX.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Topo slopes in y-direction
+# ---------------------------------------------------------
+
+default_single.TopoSlopesY.Type = "Constant"
+default_single.TopoSlopesY.GeomNames = ""
+default_single.TopoSlopesY.Geom.domain.Value = 0.0
+
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
+# mannings roughnesses do not figure into the impes (fully sat) case but we still
+# need a key for them
+
+default_single.Mannings.Type = "Constant"
+default_single.Mannings.GeomNames = ""
+default_single.Mannings.Geom.domain.Value = 0.0
+
+# -----------------------------------------------------------------------------
+# Phase sources:
+# -----------------------------------------------------------------------------
+
+default_single.PhaseSources.water.Type = "Constant"
+default_single.PhaseSources.water.GeomNames = "background"
+default_single.PhaseSources.water.Geom.background.Value = 0.0
+
+default_single.PhaseConcen.water.tce.Type = "Constant"
+default_single.PhaseConcen.water.tce.GeomNames = "concen_region"
+default_single.PhaseConcen.water.tce.Geom.concen_region.Value = 0.8
+
+default_single.Solver.PrintVelocities = True
+
+# -----------------------------------------------------------------------------
+# The Solver Impes MaxIter default value changed so to get previous
+# results we need to set it back to what it was
+# -----------------------------------------------------------------------------
+
+default_single.Solver.MaxIter = 5
+default_single.Solver.AbsTol = 1e-25
+
+
+# -----------------------------------------------------------------------------
+#
+# -----------------------------------------------------------------------------
+new_output_dir_name = get_absolute_path("test_output/default_single_input_porosity")
+mkdir(new_output_dir_name)
+cp("../input/default_single_input_porosity.pfb", new_output_dir_name)
+
+default_single.dist(new_output_dir_name + "/default_single_input_porosity.pfb")
+
+# -----------------------------------------------------------------------------
+# Run and Unload the ParFlow output files
+# -----------------------------------------------------------------------------
+default_single.run(working_directory=new_output_dir_name)
+
+# -----------------------------------------------------------------------------
+# Check results for regression
+# -----------------------------------------------------------------------------
+
+passed = True
+
+i = 0
+timestep = str(i).rjust(5, "0")
+
+sig_digits = 4
+abs_value = 1e-12
+
+test_files = ["press"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {filename}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {filename}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+test_files = ["phasex", "phasey", "phasez"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.0.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {filename}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+
+test_files = ["concen.0.00"]
+for i in range(0, 5):
+    timestep = str(i).rjust(5, "0")
+    for test_file in test_files:
+        filename = f"/{run_name}.out.{test_file}.{timestep}.pfsb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in {test_file}",
+            sig_digits,
+        ):
+            passed = False
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/forsyth2.py b/test/python/forsyth2.py
index fa1ae1d72..846c28448 100644
--- a/test/python/forsyth2.py
+++ b/test/python/forsyth2.py
@@ -1,28 +1,29 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 #  This runs Problem 2 in the paper
 #     "Robust Numerical Methods for Saturated-Unsaturated Flow with
 #      Dry Initial Conditions", Forsyth, Wu and Pruess,
 #      Advances in Water Resources, 1995.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import cp, mkdir, chdir, get_absolute_path
 from parflow.tools import settings
+from parflow.tools.compare import pf_test_file
 
-forsyth2 = Run("forsyth2", __file__)
+run_name = "forsyth2"
+forsyth2 = Run(run_name, __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Copy solid file
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/forsyth2')
+dir_name = get_absolute_path("test_output/forsyth2")
 mkdir(dir_name)
-chdir(dir_name)
-settings.set_working_directory()
 
-cp('$PF_SRC/test/input/fors2_hf.pfsol')
+cp("$PF_SRC/test/input/fors2_hf.pfsol", dir_name)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 forsyth2.FileVersion = 4
 
@@ -30,9 +31,9 @@
 forsyth2.Process.Topology.Q = 1
 forsyth2.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 forsyth2.ComputationalGrid.Lower.X = 0.0
 forsyth2.ComputationalGrid.Lower.Y = 0.0
@@ -58,20 +59,20 @@
 forsyth2.ComputationalGrid.DY = (UpperY - LowerY) / NY
 forsyth2.ComputationalGrid.DZ = (UpperZ - LowerZ) / NZ
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 Zones = "zone1 zone2 zone3above4 zone3left4 zone3right4 zone3below4 zone4"
 
-forsyth2.GeomInput.Names = f'solidinput {Zones} background'
+forsyth2.GeomInput.Names = f"solidinput {Zones} background"
 
-forsyth2.GeomInput.solidinput.InputType = 'SolidFile'
-forsyth2.GeomInput.solidinput.GeomNames = 'domain'
-forsyth2.GeomInput.solidinput.FileName = 'fors2_hf.pfsol'
+forsyth2.GeomInput.solidinput.InputType = "SolidFile"
+forsyth2.GeomInput.solidinput.GeomNames = "domain"
+forsyth2.GeomInput.solidinput.FileName = "fors2_hf.pfsol"
 
-forsyth2.GeomInput.zone1.InputType = 'Box'
-forsyth2.GeomInput.zone1.GeomName = 'zone1'
+forsyth2.GeomInput.zone1.InputType = "Box"
+forsyth2.GeomInput.zone1.GeomName = "zone1"
 
 forsyth2.Geom.zone1.Lower.X = 0.0
 forsyth2.Geom.zone1.Lower.Y = 0.0
@@ -80,8 +81,8 @@
 forsyth2.Geom.zone1.Upper.Y = 1.0
 forsyth2.Geom.zone1.Upper.Z = 650.0
 
-forsyth2.GeomInput.zone2.InputType = 'Box'
-forsyth2.GeomInput.zone2.GeomName = 'zone2'
+forsyth2.GeomInput.zone2.InputType = "Box"
+forsyth2.GeomInput.zone2.GeomName = "zone2"
 
 forsyth2.Geom.zone2.Lower.X = 0.0
 forsyth2.Geom.zone2.Lower.Y = 0.0
@@ -90,8 +91,8 @@
 forsyth2.Geom.zone2.Upper.Y = 1.0
 forsyth2.Geom.zone2.Upper.Z = 610.0
 
-forsyth2.GeomInput.zone3above4.InputType = 'Box'
-forsyth2.GeomInput.zone3above4.GeomName = 'zone3above4'
+forsyth2.GeomInput.zone3above4.InputType = "Box"
+forsyth2.GeomInput.zone3above4.GeomName = "zone3above4"
 
 forsyth2.Geom.zone3above4.Lower.X = 0.0
 forsyth2.Geom.zone3above4.Lower.Y = 0.0
@@ -100,8 +101,8 @@
 forsyth2.Geom.zone3above4.Upper.Y = 1.0
 forsyth2.Geom.zone3above4.Upper.Z = 560.0
 
-forsyth2.GeomInput.zone3left4.InputType = 'Box'
-forsyth2.GeomInput.zone3left4.GeomName = 'zone3left4'
+forsyth2.GeomInput.zone3left4.InputType = "Box"
+forsyth2.GeomInput.zone3left4.GeomName = "zone3left4"
 
 forsyth2.Geom.zone3left4.Lower.X = 0.0
 forsyth2.Geom.zone3left4.Lower.Y = 0.0
@@ -110,8 +111,8 @@
 forsyth2.Geom.zone3left4.Upper.Y = 1.0
 forsyth2.Geom.zone3left4.Upper.Z = 500.0
 
-forsyth2.GeomInput.zone3right4.InputType = 'Box'
-forsyth2.GeomInput.zone3right4.GeomName = 'zone3right4'
+forsyth2.GeomInput.zone3right4.InputType = "Box"
+forsyth2.GeomInput.zone3right4.GeomName = "zone3right4"
 
 forsyth2.Geom.zone3right4.Lower.X = 300.0
 forsyth2.Geom.zone3right4.Lower.Y = 0.0
@@ -120,8 +121,8 @@
 forsyth2.Geom.zone3right4.Upper.Y = 1.0
 forsyth2.Geom.zone3right4.Upper.Z = 500.0
 
-forsyth2.GeomInput.zone3below4.InputType = 'Box'
-forsyth2.GeomInput.zone3below4.GeomName = 'zone3below4'
+forsyth2.GeomInput.zone3below4.InputType = "Box"
+forsyth2.GeomInput.zone3below4.GeomName = "zone3below4"
 
 forsyth2.Geom.zone3below4.Lower.X = 0.0
 forsyth2.Geom.zone3below4.Lower.Y = 0.0
@@ -130,8 +131,8 @@
 forsyth2.Geom.zone3below4.Upper.Y = 1.0
 forsyth2.Geom.zone3below4.Upper.Z = 400.0
 
-forsyth2.GeomInput.zone4.InputType = 'Box'
-forsyth2.GeomInput.zone4.GeomName = 'zone4'
+forsyth2.GeomInput.zone4.InputType = "Box"
+forsyth2.GeomInput.zone4.GeomName = "zone4"
 
 forsyth2.Geom.zone4.Lower.X = 100.0
 forsyth2.Geom.zone4.Lower.Y = 0.0
@@ -140,8 +141,8 @@
 forsyth2.Geom.zone4.Upper.Y = 1.0
 forsyth2.Geom.zone4.Upper.Z = 500.0
 
-forsyth2.GeomInput.background.InputType = 'Box'
-forsyth2.GeomInput.background.GeomName = 'background'
+forsyth2.GeomInput.background.InputType = "Box"
+forsyth2.GeomInput.background.GeomName = "background"
 
 forsyth2.Geom.background.Lower.X = -99999999.0
 forsyth2.Geom.background.Lower.Y = -99999999.0
@@ -150,93 +151,95 @@
 forsyth2.Geom.background.Upper.Y = 99999999.0
 forsyth2.Geom.background.Upper.Z = 99999999.0
 
-forsyth2.Geom.domain.Patches = 'infiltration z_upper x_lower y_lower x_upper y_upper z_lower'
+forsyth2.Geom.domain.Patches = (
+    "infiltration z_upper x_lower y_lower x_upper y_upper z_lower"
+)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 forsyth2.Geom.Perm.Names = Zones
 
 # Values in cm^2
 
-forsyth2.Geom.zone1.Perm.Type = 'Constant'
+forsyth2.Geom.zone1.Perm.Type = "Constant"
 forsyth2.Geom.zone1.Perm.Value = 9.1496e-5
 
-forsyth2.Geom.zone2.Perm.Type = 'Constant'
+forsyth2.Geom.zone2.Perm.Type = "Constant"
 forsyth2.Geom.zone2.Perm.Value = 5.4427e-5
 
-forsyth2.Geom.zone3above4.Perm.Type = 'Constant'
+forsyth2.Geom.zone3above4.Perm.Type = "Constant"
 forsyth2.Geom.zone3above4.Perm.Value = 4.8033e-5
 
-forsyth2.Geom.zone3left4.Perm.Type = 'Constant'
+forsyth2.Geom.zone3left4.Perm.Type = "Constant"
 forsyth2.Geom.zone3left4.Perm.Value = 4.8033e-5
 
-forsyth2.Geom.zone3right4.Perm.Type = 'Constant'
+forsyth2.Geom.zone3right4.Perm.Type = "Constant"
 forsyth2.Geom.zone3right4.Perm.Value = 4.8033e-5
 
-forsyth2.Geom.zone3below4.Perm.Type = 'Constant'
+forsyth2.Geom.zone3below4.Perm.Type = "Constant"
 forsyth2.Geom.zone3below4.Perm.Value = 4.8033e-5
 
-forsyth2.Geom.zone4.Perm.Type = 'Constant'
+forsyth2.Geom.zone4.Perm.Type = "Constant"
 forsyth2.Geom.zone4.Perm.Value = 4.8033e-4
 
-forsyth2.Perm.TensorType = 'TensorByGeom'
+forsyth2.Perm.TensorType = "TensorByGeom"
 
-forsyth2.Geom.Perm.TensorByGeom.Names = 'background'
+forsyth2.Geom.Perm.TensorByGeom.Names = "background"
 
 forsyth2.Geom.background.Perm.TensorValX = 1.0
 forsyth2.Geom.background.Perm.TensorValY = 1.0
 forsyth2.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-forsyth2.SpecificStorage.Type = 'Constant'
-forsyth2.SpecificStorage.GeomNames = 'domain'
+forsyth2.SpecificStorage.Type = "Constant"
+forsyth2.SpecificStorage.GeomNames = "domain"
 forsyth2.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-forsyth2.Phase.Names = 'water'
+forsyth2.Phase.Names = "water"
 
-forsyth2.Phase.water.Density.Type = 'Constant'
+forsyth2.Phase.water.Density.Type = "Constant"
 forsyth2.Phase.water.Density.Value = 1.0
 
-forsyth2.Phase.water.Viscosity.Type = 'Constant'
+forsyth2.Phase.water.Viscosity.Type = "Constant"
 forsyth2.Phase.water.Viscosity.Value = 1.124e-2
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-forsyth2.Contaminants.Names = 'tce'
+forsyth2.Contaminants.Names = "tce"
 forsyth2.Contaminants.tce.Degradation.Value = 0.0
 
-forsyth2.PhaseConcen.water.tce.Type = 'Constant'
-forsyth2.PhaseConcen.water.tce.GeomNames = 'domain'
+forsyth2.PhaseConcen.water.tce.Type = "Constant"
+forsyth2.PhaseConcen.water.tce.GeomNames = "domain"
 forsyth2.PhaseConcen.water.tce.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-forsyth2.Geom.Retardation.GeomNames = 'background'
-forsyth2.Geom.background.tce.Retardation.Type = 'Linear'
+forsyth2.Geom.Retardation.GeomNames = "background"
+forsyth2.Geom.background.tce.Retardation.Type = "Linear"
 forsyth2.Geom.background.tce.Retardation.Rate = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 forsyth2.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 forsyth2.TimingInfo.BaseUnit = 1.0
 forsyth2.TimingInfo.StartCount = 0
@@ -245,47 +248,47 @@
 forsyth2.TimingInfo.StopTime = 8640.0
 
 forsyth2.TimingInfo.DumpInterval = -1
-forsyth2.TimeStep.Type = 'Constant'
+forsyth2.TimeStep.Type = "Constant"
 forsyth2.TimeStep.Value = 8640.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 forsyth2.Geom.Porosity.GeomNames = Zones
 
-forsyth2.Geom.zone1.Porosity.Type = 'Constant'
+forsyth2.Geom.zone1.Porosity.Type = "Constant"
 forsyth2.Geom.zone1.Porosity.Value = 0.3680
 
-forsyth2.Geom.zone2.Porosity.Type = 'Constant'
+forsyth2.Geom.zone2.Porosity.Type = "Constant"
 forsyth2.Geom.zone2.Porosity.Value = 0.3510
 
-forsyth2.Geom.zone3above4.Porosity.Type = 'Constant'
+forsyth2.Geom.zone3above4.Porosity.Type = "Constant"
 forsyth2.Geom.zone3above4.Porosity.Value = 0.3250
 
-forsyth2.Geom.zone3left4.Porosity.Type = 'Constant'
+forsyth2.Geom.zone3left4.Porosity.Type = "Constant"
 forsyth2.Geom.zone3left4.Porosity.Value = 0.3250
 
-forsyth2.Geom.zone3right4.Porosity.Type = 'Constant'
+forsyth2.Geom.zone3right4.Porosity.Type = "Constant"
 forsyth2.Geom.zone3right4.Porosity.Value = 0.3250
 
-forsyth2.Geom.zone3below4.Porosity.Type = 'Constant'
+forsyth2.Geom.zone3below4.Porosity.Type = "Constant"
 forsyth2.Geom.zone3below4.Porosity.Value = 0.3250
 
-forsyth2.Geom.zone4.Porosity.Type = 'Constant'
+forsyth2.Geom.zone4.Porosity.Type = "Constant"
 forsyth2.Geom.zone4.Porosity.Value = 0.3250
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-forsyth2.Domain.GeomName = 'domain'
+forsyth2.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-forsyth2.Phase.RelPerm.Type = 'VanGenuchten'
+forsyth2.Phase.RelPerm.Type = "VanGenuchten"
 forsyth2.Phase.RelPerm.GeomNames = Zones
 
 forsyth2.Geom.zone1.RelPerm.Alpha = 0.0334
@@ -309,11 +312,11 @@
 forsyth2.Geom.zone4.RelPerm.Alpha = 0.0345
 forsyth2.Geom.zone4.RelPerm.N = 1.573
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-forsyth2.Phase.Saturation.Type = 'VanGenuchten'
+forsyth2.Phase.Saturation.Type = "VanGenuchten"
 forsyth2.Phase.Saturation.GeomNames = Zones
 
 forsyth2.Geom.zone1.Saturation.Alpha = 0.0334
@@ -356,108 +359,108 @@
 forsyth2.Geom.zone4.Saturation.SRes = 0.2643
 forsyth2.Geom.zone4.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-forsyth2.Wells.Names = ''
+forsyth2.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-forsyth2.Cycle.Names = 'constant'
-forsyth2.Cycle.constant.Names = 'alltime'
+forsyth2.Cycle.Names = "constant"
+forsyth2.Cycle.constant.Names = "alltime"
 forsyth2.Cycle.constant.alltime.Length = 1
 forsyth2.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 forsyth2.BCPressure.PatchNames = forsyth2.Geom.domain.Patches
 
-forsyth2.Patch.infiltration.BCPressure.Type = 'FluxConst'
-forsyth2.Patch.infiltration.BCPressure.Cycle = 'constant'
+forsyth2.Patch.infiltration.BCPressure.Type = "FluxConst"
+forsyth2.Patch.infiltration.BCPressure.Cycle = "constant"
 forsyth2.Patch.infiltration.BCPressure.alltime.Value = -2.3148e-5
 
-forsyth2.Patch.x_lower.BCPressure.Type = 'FluxConst'
-forsyth2.Patch.x_lower.BCPressure.Cycle = 'constant'
+forsyth2.Patch.x_lower.BCPressure.Type = "FluxConst"
+forsyth2.Patch.x_lower.BCPressure.Cycle = "constant"
 forsyth2.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-forsyth2.Patch.y_lower.BCPressure.Type = 'FluxConst'
-forsyth2.Patch.y_lower.BCPressure.Cycle = 'constant'
+forsyth2.Patch.y_lower.BCPressure.Type = "FluxConst"
+forsyth2.Patch.y_lower.BCPressure.Cycle = "constant"
 forsyth2.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-forsyth2.Patch.z_lower.BCPressure.Type = 'FluxConst'
-forsyth2.Patch.z_lower.BCPressure.Cycle = 'constant'
+forsyth2.Patch.z_lower.BCPressure.Type = "FluxConst"
+forsyth2.Patch.z_lower.BCPressure.Cycle = "constant"
 forsyth2.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-forsyth2.Patch.x_upper.BCPressure.Type = 'FluxConst'
-forsyth2.Patch.x_upper.BCPressure.Cycle = 'constant'
+forsyth2.Patch.x_upper.BCPressure.Type = "FluxConst"
+forsyth2.Patch.x_upper.BCPressure.Cycle = "constant"
 forsyth2.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-forsyth2.Patch.y_upper.BCPressure.Type = 'FluxConst'
-forsyth2.Patch.y_upper.BCPressure.Cycle = 'constant'
+forsyth2.Patch.y_upper.BCPressure.Type = "FluxConst"
+forsyth2.Patch.y_upper.BCPressure.Cycle = "constant"
 forsyth2.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-forsyth2.Patch.z_upper.BCPressure.Type = 'FluxConst'
-forsyth2.Patch.z_upper.BCPressure.Cycle = 'constant'
+forsyth2.Patch.z_upper.BCPressure.Type = "FluxConst"
+forsyth2.Patch.z_upper.BCPressure.Cycle = "constant"
 forsyth2.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-forsyth2.TopoSlopesX.Type = 'Constant'
-forsyth2.TopoSlopesX.GeomNames = 'domain'
+forsyth2.TopoSlopesX.Type = "Constant"
+forsyth2.TopoSlopesX.GeomNames = "domain"
 
 forsyth2.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-forsyth2.TopoSlopesY.Type = 'Constant'
-forsyth2.TopoSlopesY.GeomNames = 'domain'
+forsyth2.TopoSlopesY.Type = "Constant"
+forsyth2.TopoSlopesY.GeomNames = "domain"
 
 forsyth2.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-forsyth2.Mannings.Type = 'Constant'
-forsyth2.Mannings.GeomNames = 'domain'
-forsyth2.Mannings.Geom.domain.Value = 0.
+forsyth2.Mannings.Type = "Constant"
+forsyth2.Mannings.GeomNames = "domain"
+forsyth2.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-forsyth2.ICPressure.Type = 'Constant'
-forsyth2.ICPressure.GeomNames = 'domain'
+forsyth2.ICPressure.Type = "Constant"
+forsyth2.ICPressure.GeomNames = "domain"
 forsyth2.Geom.domain.ICPressure.Value = -734.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-forsyth2.PhaseSources.water.Type = 'Constant'
-forsyth2.PhaseSources.water.GeomNames = 'background'
+forsyth2.PhaseSources.water.Type = "Constant"
+forsyth2.PhaseSources.water.GeomNames = "background"
 forsyth2.PhaseSources.water.Geom.background.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-forsyth2.KnownSolution = 'NoKnownSolution'
+forsyth2.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-forsyth2.Solver = 'Richards'
+forsyth2.Solver = "Richards"
 forsyth2.Solver.MaxIter = 10000
 
 forsyth2.Solver.Nonlinear.MaxIter = 15
@@ -470,12 +473,52 @@
 forsyth2.Solver.Linear.KrylovDimension = 25
 forsyth2.Solver.Linear.MaxRestarts = 2
 
-forsyth2.Solver.Linear.Preconditioner = 'MGSemi'
+forsyth2.Solver.Linear.Preconditioner = "MGSemi"
 forsyth2.Solver.Linear.Preconditioner.MGSemi.MaxIter = 1
 forsyth2.Solver.Linear.Preconditioner.MGSemi.MaxLevels = 100
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-forsyth2.run()
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = dir_name
+correct_output_dir_name = get_absolute_path("../correct_output")
+forsyth2.run(working_directory=dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+
+
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+for i in range(2):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/harvey_flow.py b/test/python/harvey_flow.py
index a52d9eb17..404d3d423 100644
--- a/test/python/harvey_flow.py
+++ b/test/python/harvey_flow.py
@@ -1,30 +1,36 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # this runs the Cape Cod site flow case for the Harvey and Garabedian bacterial
 # injection experiment from Maxwell, et al, 2007.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
+import os, sys
 from parflow import Run
-from parflow.tools.fs import cp, chdir, mkdir, get_absolute_path
+from parflow.tools.fs import cp, chdir, mkdir, get_absolute_path, rm
+from parflow.tools.io import read_pfb, write_pfb, ParflowBinaryReader
+from parflow.tools.compare import pf_test_file
+from parflow.tools.hydrology import compute_hydraulic_head
 
-hflow = Run("hflow", __file__)
+run_name = "harvey_flow.1"
 
-#-----------------------------------------------------------------------------
+hflow = Run(run_name, __file__)
+
+# -----------------------------------------------------------------------------
 # File input version number
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 hflow.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Process Topology
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 hflow.Process.Topology.P = 1
 hflow.Process.Topology.Q = 1
 hflow.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 hflow.ComputationalGrid.Lower.X = 0.0
 hflow.ComputationalGrid.Lower.Y = 0.0
@@ -38,22 +44,22 @@
 hflow.ComputationalGrid.NY = 30
 hflow.ComputationalGrid.NZ = 100
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # The Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.GeomInput.Names = 'domain_input upper_aquifer_input lower_aquifer_input'
+hflow.GeomInput.Names = "domain_input upper_aquifer_input lower_aquifer_input"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.GeomInput.domain_input.InputType = 'Box'
-hflow.GeomInput.domain_input.GeomName = 'domain'
+hflow.GeomInput.domain_input.InputType = "Box"
+hflow.GeomInput.domain_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 hflow.Geom.domain.Lower.X = 0.0
 hflow.Geom.domain.Lower.Y = 0.0
@@ -63,18 +69,18 @@
 hflow.Geom.domain.Upper.Y = 10.2
 hflow.Geom.domain.Upper.Z = 3.8
 
-hflow.Geom.domain.Patches = 'left right front back bottom top'
+hflow.Geom.domain.Patches = "left right front back bottom top"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Upper Aquifer Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.GeomInput.upper_aquifer_input.InputType = 'Box'
-hflow.GeomInput.upper_aquifer_input.GeomName = 'upper_aquifer'
+hflow.GeomInput.upper_aquifer_input.InputType = "Box"
+hflow.GeomInput.upper_aquifer_input.GeomName = "upper_aquifer"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Upper Aquifer Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 hflow.Geom.upper_aquifer.Lower.X = 0.0
 hflow.Geom.upper_aquifer.Lower.Y = 0.0
@@ -84,16 +90,16 @@
 hflow.Geom.upper_aquifer.Upper.Y = 10.2
 hflow.Geom.upper_aquifer.Upper.Z = 3.8
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Lower Aquifer Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.GeomInput.lower_aquifer_input.InputType = 'Box'
-hflow.GeomInput.lower_aquifer_input.GeomName = 'lower_aquifer'
+hflow.GeomInput.lower_aquifer_input.InputType = "Box"
+hflow.GeomInput.lower_aquifer_input.GeomName = "lower_aquifer"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Lower Aquifer Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 hflow.Geom.lower_aquifer.Lower.X = 0.0
 hflow.Geom.lower_aquifer.Lower.Y = 0.0
@@ -103,17 +109,17 @@
 hflow.Geom.lower_aquifer.Upper.Y = 10.2
 hflow.Geom.lower_aquifer.Upper.Z = 1.5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.Geom.Perm.Names = 'upper_aquifer lower_aquifer'
+hflow.Geom.Perm.Names = "upper_aquifer lower_aquifer"
 
 # we open a file, in this case from PEST to set upper and lower kg and sigma
-cp('$PF_SRC/test/input/stats4.txt')
-chdir('.')
+cp("$PF_SRC/test/input/stats4.txt")
+chdir(".")
 
-file = open('stats4.txt', 'r')
+file = open("stats4.txt", "r")
 lines = file.readlines()
 kgu = float(lines[0])
 varu = float(lines[1])
@@ -123,7 +129,7 @@
 ## we use the parallel turning bands formulation in ParFlow to simulate
 ## GRF for upper and lower aquifer
 
-hflow.Geom.upper_aquifer.Perm.Type = 'TurnBands'
+hflow.Geom.upper_aquifer.Perm.Type = "TurnBands"
 hflow.Geom.upper_aquifer.Perm.LambdaX = 3.60
 hflow.Geom.upper_aquifer.Perm.LambdaY = 3.60
 hflow.Geom.upper_aquifer.Perm.LambdaZ = 0.19
@@ -136,9 +142,9 @@
 hflow.Geom.upper_aquifer.Perm.KMax = 100.0000001
 hflow.Geom.upper_aquifer.Perm.DelK = 0.2
 hflow.Geom.upper_aquifer.Perm.Seed = 33333
-hflow.Geom.upper_aquifer.Perm.LogNormal = 'Log'
-hflow.Geom.upper_aquifer.Perm.StratType = 'Bottom'
-hflow.Geom.lower_aquifer.Perm.Type = 'TurnBands'
+hflow.Geom.upper_aquifer.Perm.LogNormal = "Log"
+hflow.Geom.upper_aquifer.Perm.StratType = "Bottom"
+hflow.Geom.lower_aquifer.Perm.Type = "TurnBands"
 hflow.Geom.lower_aquifer.Perm.LambdaX = 3.60
 hflow.Geom.lower_aquifer.Perm.LambdaY = 3.60
 hflow.Geom.lower_aquifer.Perm.LambdaZ = 0.19
@@ -151,28 +157,28 @@
 hflow.Geom.lower_aquifer.Perm.KMax = 100.0000001
 hflow.Geom.lower_aquifer.Perm.DelK = 0.2
 hflow.Geom.lower_aquifer.Perm.Seed = 33333
-hflow.Geom.lower_aquifer.Perm.LogNormal = 'Log'
-hflow.Geom.lower_aquifer.Perm.StratType = 'Bottom'
+hflow.Geom.lower_aquifer.Perm.LogNormal = "Log"
+hflow.Geom.lower_aquifer.Perm.StratType = "Bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # uncomment the lines below to run parallel gaussian instead
 # of parallel turning bands
 
-#hflow.Geom.upper_aquifer.Perm.Type = "ParGauss"
+# hflow.Geom.upper_aquifer.Perm.Type = "ParGauss"
 
-#hflow.Geom.upper_aquifer.Perm.Seed = 1
-#hflow.Geom.upper_aquifer.Perm.MaxNPts = 70.0
-#hflow.Geom.upper_aquifer.Perm.MaxCpts = 20
+# hflow.Geom.upper_aquifer.Perm.Seed = 1
+# hflow.Geom.upper_aquifer.Perm.MaxNPts = 70.0
+# hflow.Geom.upper_aquifer.Perm.MaxCpts = 20
 
 
-#hflow.Geom.lower_aquifer.Perm.Type = "ParGauss"
+# hflow.Geom.lower_aquifer.Perm.Type = "ParGauss"
 
-#hflow.Geom.lower_aquifer.Perm.Seed = 1
-#hflow.Geom.lower_aquifer.Perm.MaxNPts = 70.0
-#pfset Geom.lower_aquifer.Perm.MaxCpts = 20
+# hflow.Geom.lower_aquifer.Perm.Seed = 1
+# hflow.Geom.lower_aquifer.Perm.MaxNPts = 70.0
+# pfset Geom.lower_aquifer.Perm.MaxCpts = 20
 
-#pfset lower aqu and upper aq stats to pest/read in values
-#-----------------------------------------------------------------------------
+# pfset lower aqu and upper aq stats to pest/read in values
+# -----------------------------------------------------------------------------
 
 hflow.Geom.upper_aquifer.Perm.GeomMean = kgu
 hflow.Geom.upper_aquifer.Perm.Sigma = varu
@@ -181,51 +187,51 @@
 hflow.Geom.lower_aquifer.Perm.Sigma = varl
 
 
-hflow.Perm.TensorType = 'TensorByGeom'
+hflow.Perm.TensorType = "TensorByGeom"
 
-hflow.Geom.Perm.TensorByGeom.Names = 'domain'
+hflow.Geom.Perm.TensorByGeom.Names = "domain"
 
 hflow.Geom.domain.Perm.TensorValX = 1.0
 hflow.Geom.domain.Perm.TensorValY = 1.0
 hflow.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # specific storage does not figure into the impes (fully sat) case but we still
 # need a key for it
 
-hflow.SpecificStorage.Type = 'Constant'
-hflow.SpecificStorage.GeomNames = ''
+hflow.SpecificStorage.Type = "Constant"
+hflow.SpecificStorage.GeomNames = ""
 hflow.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.Phase.Names = 'water'
+hflow.Phase.Names = "water"
 
-hflow.Phase.water.Density.Type = 'Constant'
+hflow.Phase.water.Density.Type = "Constant"
 hflow.Phase.water.Density.Value = 1.0
 
-hflow.Phase.water.Viscosity.Type = 'Constant'
+hflow.Phase.water.Viscosity.Type = "Constant"
 hflow.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.Contaminants.Names = ''
+hflow.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 hflow.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 hflow.TimingInfo.BaseUnit = 1.0
 hflow.TimingInfo.StartCount = -1
@@ -233,125 +239,161 @@
 hflow.TimingInfo.StopTime = 0.0
 hflow.TimingInfo.DumpInterval = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.Geom.Porosity.GeomNames = 'domain'
-hflow.Geom.domain.Porosity.Type = 'Constant'
+hflow.Geom.Porosity.GeomNames = "domain"
+hflow.Geom.domain.Porosity.Type = "Constant"
 hflow.Geom.domain.Porosity.Value = 0.390
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.Domain.GeomName = 'domain'
+hflow.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mobility
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.Phase.water.Mobility.Type = 'Constant'
+hflow.Phase.water.Mobility.Type = "Constant"
 hflow.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.Wells.Names = ''
+hflow.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.Cycle.Names = 'constant'
-hflow.Cycle.constant.Names = 'alltime'
+hflow.Cycle.Names = "constant"
+hflow.Cycle.constant.Names = "alltime"
 hflow.Cycle.constant.alltime.Length = 1
 hflow.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.BCPressure.PatchNames = 'left right front back bottom top'
+hflow.BCPressure.PatchNames = "left right front back bottom top"
 
-hflow.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-hflow.Patch.left.BCPressure.Cycle = 'constant'
-hflow.Patch.left.BCPressure.RefGeom = 'domain'
-hflow.Patch.left.BCPressure.RefPatch = 'bottom'
+hflow.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+hflow.Patch.left.BCPressure.Cycle = "constant"
+hflow.Patch.left.BCPressure.RefGeom = "domain"
+hflow.Patch.left.BCPressure.RefPatch = "bottom"
 hflow.Patch.left.BCPressure.alltime.Value = 10.0
 
-hflow.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-hflow.Patch.right.BCPressure.Cycle = 'constant'
-hflow.Patch.right.BCPressure.RefGeom = 'domain'
-hflow.Patch.right.BCPressure.RefPatch = 'bottom'
+hflow.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+hflow.Patch.right.BCPressure.Cycle = "constant"
+hflow.Patch.right.BCPressure.RefGeom = "domain"
+hflow.Patch.right.BCPressure.RefPatch = "bottom"
 hflow.Patch.right.BCPressure.alltime.Value = 9.97501
 
-hflow.Patch.front.BCPressure.Type = 'FluxConst'
-hflow.Patch.front.BCPressure.Cycle = 'constant'
+hflow.Patch.front.BCPressure.Type = "FluxConst"
+hflow.Patch.front.BCPressure.Cycle = "constant"
 hflow.Patch.front.BCPressure.alltime.Value = 0.0
 
-hflow.Patch.back.BCPressure.Type = 'FluxConst'
-hflow.Patch.back.BCPressure.Cycle = 'constant'
+hflow.Patch.back.BCPressure.Type = "FluxConst"
+hflow.Patch.back.BCPressure.Cycle = "constant"
 hflow.Patch.back.BCPressure.alltime.Value = 0.0
 
-hflow.Patch.bottom.BCPressure.Type = 'FluxConst'
-hflow.Patch.bottom.BCPressure.Cycle = 'constant'
+hflow.Patch.bottom.BCPressure.Type = "FluxConst"
+hflow.Patch.bottom.BCPressure.Cycle = "constant"
 hflow.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-hflow.Patch.top.BCPressure.Type = 'FluxConst'
-hflow.Patch.top.BCPressure.Cycle = 'constant'
+hflow.Patch.top.BCPressure.Type = "FluxConst"
+hflow.Patch.top.BCPressure.Cycle = "constant"
 hflow.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # topo slopes do not figure into the impes (fully sat) case but we still
 # need keys for them
 
-hflow.TopoSlopesX.Type = 'Constant'
-hflow.TopoSlopesX.GeomNames = 'domain'
+hflow.TopoSlopesX.Type = "Constant"
+hflow.TopoSlopesX.GeomNames = ""
 hflow.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-hflow.TopoSlopesY.Type = 'Constant'
-hflow.TopoSlopesY.GeomNames = 'domain'
+hflow.TopoSlopesY.Type = "Constant"
+hflow.TopoSlopesY.GeomNames = ""
 
 hflow.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # mannings roughnesses do not figure into the impes (fully sat) case but we still
 # need a key for them
 
-hflow.Mannings.Type = 'Constant'
-hflow.Mannings.GeomNames = 'domain'
-hflow.Mannings.Geom.domain.Value = 0.
+hflow.Mannings.Type = "Constant"
+hflow.Mannings.GeomNames = ""
+hflow.Mannings.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.PhaseSources.water.Type = 'Constant'
-hflow.PhaseSources.water.GeomNames = 'domain'
+hflow.PhaseSources.water.Type = "Constant"
+hflow.PhaseSources.water.GeomNames = "domain"
 hflow.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 #  Solver Impes
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 hflow.Solver.MaxIter = 50
-hflow.Solver.AbsTol = 1E-10
-hflow.Solver.Drop = 1E-15
+hflow.Solver.AbsTol = 1e-10
+hflow.Solver.Drop = 1e-15
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/hflow')
+# -----------------------------------------------------------------------------
+hflow.Geom.upper_aquifer.Perm.Seed = 33335
+hflow.Geom.lower_aquifer.Perm.Seed = 31315
+dir_name = get_absolute_path("test_output/harvey_flow")
 mkdir(dir_name)
+correct_output_dir_name = get_absolute_path("../correct_output")
 hflow.run(working_directory=dir_name)
+
+passed = True
+
+test_files = ["porosity", "perm_x", "perm_y", "perm_z", "press"]
+for test_file in test_files:
+    filename = f"{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        os.path.join(dir_name, filename),
+        os.path.join(correct_output_dir_name, filename),
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+press_path = os.path.join(dir_name, "harvey_flow.1.out.press.pfb")
+pressure = read_pfb(press_path)
+header = ParflowBinaryReader(press_path).read_header()
+z, dz = header["z"], header["dz"]
+hhead = compute_hydraulic_head(pressure, z, dz)
+write_pfb(os.path.join(dir_name, "harvey_flow.1.head.pfb"), hhead)
+
+filename = "harvey_flow.1.head.pfb"
+if not pf_test_file(
+    os.path.join(dir_name, filename),
+    os.path.join(correct_output_dir_name, filename),
+    f"Max difference in hydraulic head",
+):
+    passed = False
+
+rm(dir_name)
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/harvey_flow_pgs.py b/test/python/harvey_flow_pgs.py
index 5458890f2..bbb8a0035 100644
--- a/test/python/harvey_flow_pgs.py
+++ b/test/python/harvey_flow_pgs.py
@@ -1,28 +1,34 @@
 # this runs the Cape Cod site flow case for the Harvey and Garabedian bacterial
 # injection experiment from Maxwell, et al, 2007.
 
+import os, sys
 from parflow import Run
-from parflow.tools.fs import cp, chdir, mkdir, get_absolute_path
+from parflow.tools.fs import cp, chdir, mkdir, get_absolute_path, rm
+from parflow.tools.io import read_pfb, write_pfb, ParflowBinaryReader
+from parflow.tools.compare import pf_test_file
+from parflow.tools.hydrology import compute_hydraulic_head
 
-hflow = Run("harvey_flow_pgs", __file__)
+run_name = "harvey_flow_pgs.1"
 
-#-----------------------------------------------------------------------------
+hflow = Run(run_name, __file__)
+
+# -----------------------------------------------------------------------------
 # File input version number
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 hflow.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Process Topology
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 hflow.Process.Topology.P = 1
 hflow.Process.Topology.Q = 1
 hflow.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 hflow.ComputationalGrid.Lower.X = 0.0
 hflow.ComputationalGrid.Lower.Y = 0.0
 hflow.ComputationalGrid.Lower.Z = 0.0
@@ -35,22 +41,22 @@
 hflow.ComputationalGrid.NY = 30
 hflow.ComputationalGrid.NZ = 100
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # The Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.GeomInput.Names = 'domain_input upper_aquifer_input lower_aquifer_input'
+hflow.GeomInput.Names = "domain_input upper_aquifer_input lower_aquifer_input"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.GeomInput.domain_input.InputType = 'Box'
-hflow.GeomInput.domain_input.GeomName = 'domain'
+hflow.GeomInput.domain_input.InputType = "Box"
+hflow.GeomInput.domain_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 hflow.Geom.domain.Lower.X = 0.0
 hflow.Geom.domain.Lower.Y = 0.0
@@ -60,18 +66,18 @@
 hflow.Geom.domain.Upper.Y = 10.2
 hflow.Geom.domain.Upper.Z = 3.8
 
-hflow.Geom.domain.Patches = 'left right front back bottom top'
+hflow.Geom.domain.Patches = "left right front back bottom top"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Upper Aquifer Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.GeomInput.upper_aquifer_input.InputType = 'Box'
-hflow.GeomInput.upper_aquifer_input.GeomName = 'upper_aquifer'
+hflow.GeomInput.upper_aquifer_input.InputType = "Box"
+hflow.GeomInput.upper_aquifer_input.GeomName = "upper_aquifer"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Upper Aquifer Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 hflow.Geom.upper_aquifer.Lower.X = 0.0
 hflow.Geom.upper_aquifer.Lower.Y = 0.0
@@ -82,16 +88,16 @@
 hflow.Geom.upper_aquifer.Upper.Y = 10.2
 hflow.Geom.upper_aquifer.Upper.Z = 3.8
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Lower Aquifer Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.GeomInput.lower_aquifer_input.InputType = 'Box'
-hflow.GeomInput.lower_aquifer_input.GeomName = 'lower_aquifer'
+hflow.GeomInput.lower_aquifer_input.InputType = "Box"
+hflow.GeomInput.lower_aquifer_input.GeomName = "lower_aquifer"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Lower Aquifer Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 hflow.Geom.lower_aquifer.Lower.X = 0.0
 hflow.Geom.lower_aquifer.Lower.Y = 0.0
@@ -101,17 +107,17 @@
 hflow.Geom.lower_aquifer.Upper.Y = 10.2
 hflow.Geom.lower_aquifer.Upper.Z = 1.5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.Geom.Perm.Names = 'upper_aquifer lower_aquifer'
+hflow.Geom.Perm.Names = "upper_aquifer lower_aquifer"
 
 # we open a file, in this case from PEST to set upper and lower kg and sigma
-cp('$PF_SRC/test/input/stats4.txt')
-chdir('.')
+cp("$PF_SRC/test/input/stats4.txt")
+chdir(".")
 
-file = open('stats4.txt', 'r')
+file = open("stats4.txt", "r")
 lines = file.readlines()
 kgu = float(lines[0])
 varu = float(lines[1])
@@ -138,8 +144,8 @@
 hflow.Geom.upper_aquifer.Perm.KMax = 100.0000001
 hflow.Geom.upper_aquifer.Perm.DelK = 0.2
 hflow.Geom.upper_aquifer.Perm.Seed = 33333
-hflow.Geom.upper_aquifer.Perm.LogNormal = 'Log'
-hflow.Geom.upper_aquifer.Perm.StratType = 'Bottom'
+hflow.Geom.upper_aquifer.Perm.LogNormal = "Log"
+hflow.Geom.upper_aquifer.Perm.StratType = "Bottom"
 
 hflow.Geom.lower_aquifer.Perm.LambdaX = 3.60
 hflow.Geom.lower_aquifer.Perm.LambdaY = 3.60
@@ -154,8 +160,8 @@
 hflow.Geom.lower_aquifer.Perm.KMax = 100.0000001
 hflow.Geom.lower_aquifer.Perm.DelK = 0.2
 hflow.Geom.lower_aquifer.Perm.Seed = 33333
-hflow.Geom.lower_aquifer.Perm.LogNormal = 'Log'
-hflow.Geom.lower_aquifer.Perm.StratType = 'Bottom'
+hflow.Geom.lower_aquifer.Perm.LogNormal = "Log"
+hflow.Geom.lower_aquifer.Perm.StratType = "Bottom"
 
 hflow.Geom.upper_aquifer.Perm.Seed = 1
 hflow.Geom.upper_aquifer.Perm.MaxNPts = 70
@@ -165,18 +171,18 @@
 hflow.Geom.lower_aquifer.Perm.MaxNPts = 70
 hflow.Geom.lower_aquifer.Perm.MaxCpts = 20
 
-hflow.Geom.lower_aquifer.Perm.Type = "TurnBands"
-hflow.Geom.upper_aquifer.Perm.Type = "TurnBands"
+# hflow.Geom.lower_aquifer.Perm.Type = "TurnBands"
+# hflow.Geom.upper_aquifer.Perm.Type = "TurnBands"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # uncomment the lines below to run parallel gaussian instead
 # of parallel turning bands
 
-# hflow.Geom.lower_aquifer.Perm.Type = 'ParGauss'
-# hflow.Geom.upper_aquifer.Perm.Type = 'ParGauss'
+hflow.Geom.lower_aquifer.Perm.Type = "ParGauss"
+hflow.Geom.upper_aquifer.Perm.Type = "ParGauss"
 
-#pfset lower aqu and upper aq stats to pest/read in values
-#-----------------------------------------------------------------------------
+# pfset lower aqu and upper aq stats to pest/read in values
+# -----------------------------------------------------------------------------
 
 hflow.Geom.upper_aquifer.Perm.GeomMean = kgu
 hflow.Geom.upper_aquifer.Perm.Sigma = varu
@@ -184,51 +190,51 @@
 hflow.Geom.lower_aquifer.Perm.GeomMean = kgl
 hflow.Geom.lower_aquifer.Perm.Sigma = varl
 
-hflow.Perm.TensorType = 'TensorByGeom'
+hflow.Perm.TensorType = "TensorByGeom"
 
-hflow.Geom.Perm.TensorByGeom.Names = 'domain'
+hflow.Geom.Perm.TensorByGeom.Names = "domain"
 
 hflow.Geom.domain.Perm.TensorValX = 1.0
 hflow.Geom.domain.Perm.TensorValY = 1.0
 hflow.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # specific storage does not figure into the impes (fully sat) case but we still
 # need a key for it
 
-hflow.SpecificStorage.Type = 'Constant'
-hflow.SpecificStorage.GeomNames = ''
+hflow.SpecificStorage.Type = "Constant"
+hflow.SpecificStorage.GeomNames = ""
 hflow.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.Phase.Names = 'water'
+hflow.Phase.Names = "water"
 
-hflow.Phase.water.Density.Type = 'Constant'
+hflow.Phase.water.Density.Type = "Constant"
 hflow.Phase.water.Density.Value = 1.0
 
-hflow.Phase.water.Viscosity.Type = 'Constant'
+hflow.Phase.water.Viscosity.Type = "Constant"
 hflow.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.Contaminants.Names = ''
+hflow.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 hflow.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 hflow.TimingInfo.BaseUnit = 1.0
 hflow.TimingInfo.StartCount = -1
@@ -236,124 +242,159 @@
 hflow.TimingInfo.StopTime = 0.0
 hflow.TimingInfo.DumpInterval = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.Geom.Porosity.GeomNames = 'domain'
+hflow.Geom.Porosity.GeomNames = "domain"
 
-hflow.Geom.domain.Porosity.Type = 'Constant'
+hflow.Geom.domain.Porosity.Type = "Constant"
 hflow.Geom.domain.Porosity.Value = 0.390
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
-hflow.Domain.GeomName = 'domain'
+# -----------------------------------------------------------------------------
+hflow.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mobility
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.Phase.water.Mobility.Type = 'Constant'
+hflow.Phase.water.Mobility.Type = "Constant"
 hflow.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.Wells.Names = ''
+hflow.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.Cycle.Names = 'constant'
-hflow.Cycle.constant.Names = 'alltime'
+hflow.Cycle.Names = "constant"
+hflow.Cycle.constant.Names = "alltime"
 hflow.Cycle.constant.alltime.Length = 1
 hflow.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.BCPressure.PatchNames = 'left right front back bottom top'
+hflow.BCPressure.PatchNames = "left right front back bottom top"
 
-hflow.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-hflow.Patch.left.BCPressure.Cycle = 'constant'
-hflow.Patch.left.BCPressure.RefGeom = 'domain'
-hflow.Patch.left.BCPressure.RefPatch = 'bottom'
+hflow.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+hflow.Patch.left.BCPressure.Cycle = "constant"
+hflow.Patch.left.BCPressure.RefGeom = "domain"
+hflow.Patch.left.BCPressure.RefPatch = "bottom"
 hflow.Patch.left.BCPressure.alltime.Value = 10.0
 
-hflow.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-hflow.Patch.right.BCPressure.Cycle = 'constant'
-hflow.Patch.right.BCPressure.RefGeom = 'domain'
-hflow.Patch.right.BCPressure.RefPatch = 'bottom'
+hflow.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+hflow.Patch.right.BCPressure.Cycle = "constant"
+hflow.Patch.right.BCPressure.RefGeom = "domain"
+hflow.Patch.right.BCPressure.RefPatch = "bottom"
 hflow.Patch.right.BCPressure.alltime.Value = 9.97501
 
-hflow.Patch.front.BCPressure.Type = 'FluxConst'
-hflow.Patch.front.BCPressure.Cycle = 'constant'
+hflow.Patch.front.BCPressure.Type = "FluxConst"
+hflow.Patch.front.BCPressure.Cycle = "constant"
 hflow.Patch.front.BCPressure.alltime.Value = 0.0
 
-hflow.Patch.back.BCPressure.Type = 'FluxConst'
-hflow.Patch.back.BCPressure.Cycle = 'constant'
+hflow.Patch.back.BCPressure.Type = "FluxConst"
+hflow.Patch.back.BCPressure.Cycle = "constant"
 hflow.Patch.back.BCPressure.alltime.Value = 0.0
 
-hflow.Patch.bottom.BCPressure.Type = 'FluxConst'
-hflow.Patch.bottom.BCPressure.Cycle = 'constant'
+hflow.Patch.bottom.BCPressure.Type = "FluxConst"
+hflow.Patch.bottom.BCPressure.Cycle = "constant"
 hflow.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-hflow.Patch.top.BCPressure.Type = 'FluxConst'
-hflow.Patch.top.BCPressure.Cycle = 'constant'
+hflow.Patch.top.BCPressure.Type = "FluxConst"
+hflow.Patch.top.BCPressure.Cycle = "constant"
 hflow.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # topo slopes do not figure into the impes (fully sat) case but we still
 # need keys for them
 
-hflow.TopoSlopesX.Type = 'Constant'
-hflow.TopoSlopesX.GeomNames = 'domain'
+hflow.TopoSlopesX.Type = "Constant"
+hflow.TopoSlopesX.GeomNames = ""
 hflow.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-hflow.TopoSlopesY.Type = 'Constant'
-hflow.TopoSlopesY.GeomNames = 'domain'
+hflow.TopoSlopesY.Type = "Constant"
+hflow.TopoSlopesY.GeomNames = ""
 hflow.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # mannings roughnesses do not figure into the impes (fully sat) case but we still
 # need a key for them
 
-hflow.Mannings.Type = 'Constant'
-hflow.Mannings.GeomNames = 'domain'
-hflow.Mannings.Geom.domain.Value = 0.
+hflow.Mannings.Type = "Constant"
+hflow.Mannings.GeomNames = ""
+hflow.Mannings.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-hflow.PhaseSources.water.Type = 'Constant'
-hflow.PhaseSources.water.GeomNames = 'domain'
+hflow.PhaseSources.water.Type = "Constant"
+hflow.PhaseSources.water.GeomNames = "domain"
 hflow.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 #  Solver Impes
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 hflow.Solver.MaxIter = 50
-hflow.Solver.AbsTol = 1E-10
-hflow.Solver.Drop = 1E-15
+hflow.Solver.AbsTol = 1e-10
+hflow.Solver.Drop = 1e-15
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/hflow_p')
+hflow.Geom.upper_aquifer.Perm.Seed = 33335
+hflow.Geom.lower_aquifer.Perm.Seed = 31315
+dir_name = get_absolute_path("test_output/harvey_flow_pgs")
 mkdir(dir_name)
+correct_output_dir_name = get_absolute_path("../correct_output")
 hflow.run(working_directory=dir_name)
+
+press_path = os.path.join(dir_name, "harvey_flow_pgs.1.out.press.pfb")
+pressure = read_pfb(press_path)
+header = ParflowBinaryReader(press_path).read_header()
+z, dz = header["z"], header["dz"]
+hhead = compute_hydraulic_head(pressure, z, dz)
+write_pfb(os.path.join(dir_name, "harvey_flow_pgs.1.head.pfb"), hhead)
+
+passed = True
+test_files = ["perm_x", "perm_y", "perm_z", "press", "porosity"]
+for test_file in test_files:
+    filename = f"{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        os.path.join(dir_name, filename),
+        os.path.join(correct_output_dir_name, filename),
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+filename = "harvey_flow_pgs.1.head.pfb"
+if not pf_test_file(
+    os.path.join(dir_name, filename),
+    os.path.join(correct_output_dir_name, filename),
+    f"Max difference in hydraulic head",
+):
+    passed = False
+
+rm(dir_name)
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/indicator_field.py b/test/python/indicator_field.py
index 62cdb5244..2d43b717b 100644
--- a/test/python/indicator_field.py
+++ b/test/python/indicator_field.py
@@ -1,25 +1,26 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Problem to test use of indicator field.
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 import sys
 from parflow import Run
 from parflow.tools.fs import cp, mkdir, chdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
-ifield = Run("indicator_field", __file__)
+run_name = "indicator_field"
+ifield = Run(run_name, __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Copying files
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/ifield')
+dir_name = get_absolute_path("test_output/ifield")
 mkdir(dir_name)
-chdir(dir_name)
 
-cp('$PF_SRC/test/input/small_domain.pfsol')
-cp('$PF_SRC/test/input/small_domain_indicator_field.pfb')
+cp("$PF_SRC/test/input/small_domain.pfsol", dir_name)
+cp("$PF_SRC/test/input/small_domain_indicator_field.pfb", dir_name)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 ifield.FileVersion = 4
 
@@ -32,9 +33,9 @@
 ifield.Process.Topology.Q = 1
 ifield.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 ifield.ComputationalGrid.Lower.X = 0.0
 ifield.ComputationalGrid.Lower.Y = 0.0
@@ -56,251 +57,253 @@
 NY = ifield.ComputationalGrid.NY
 NZ = ifield.ComputationalGrid.NZ
 
-ifield.ComputationalGrid.DX = (UpperX - LowerX)/NX
-ifield.ComputationalGrid.DY = (UpperY - LowerY)/NY
-ifield.ComputationalGrid.DZ = (UpperZ - LowerZ)/NZ
+ifield.ComputationalGrid.DX = (UpperX - LowerX) / NX
+ifield.ComputationalGrid.DY = (UpperY - LowerY) / NY
+ifield.ComputationalGrid.DZ = (UpperZ - LowerZ) / NZ
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-ifield.GeomInput.Names = 'solid_input indicator_input'
+ifield.GeomInput.Names = "solid_input indicator_input"
 
-ifield.GeomInput.solid_input.InputType = 'SolidFile'
-ifield.GeomInput.solid_input.GeomNames = 'domain'
-ifield.GeomInput.solid_input.FileName = 'small_domain.pfsol'
+ifield.GeomInput.solid_input.InputType = "SolidFile"
+ifield.GeomInput.solid_input.GeomNames = "domain"
+ifield.GeomInput.solid_input.FileName = "small_domain.pfsol"
 
-ifield.Geom.domain.Patches = 'infiltration z_upper x_lower y_lower x_upper y_upper z_lower'
+ifield.Geom.domain.Patches = (
+    "infiltration z_upper x_lower y_lower x_upper y_upper z_lower"
+)
 
-ifield.GeomInput.indicator_input.InputType = 'IndicatorField'
-ifield.GeomInput.indicator_input.GeomNames = 'indicator'
-ifield.Geom.indicator_input.FileName = 'small_domain_indicator_field.pfb'
+ifield.GeomInput.indicator_input.InputType = "IndicatorField"
+ifield.GeomInput.indicator_input.GeomNames = "indicator"
+ifield.Geom.indicator_input.FileName = "small_domain_indicator_field.pfb"
 
 ifield.GeomInput.indicator.Value = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-ifield.Geom.Perm.Names = 'domain'
+ifield.Geom.Perm.Names = "domain"
 
-ifield.Geom.domain.Perm.Type = 'Constant'
+ifield.Geom.domain.Perm.Type = "Constant"
 ifield.Geom.domain.Perm.Value = 1.0
 
-ifield.Perm.TensorType = 'TensorByGeom'
+ifield.Perm.TensorType = "TensorByGeom"
 
-ifield.Geom.Perm.TensorByGeom.Names = 'domain'
+ifield.Geom.Perm.TensorByGeom.Names = "domain"
 
 ifield.Geom.domain.Perm.TensorValX = 1.0
 ifield.Geom.domain.Perm.TensorValY = 1.0
 ifield.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-ifield.SpecificStorage.Type = 'Constant'
-ifield.SpecificStorage.GeomNames = 'domain'
+ifield.SpecificStorage.Type = "Constant"
+ifield.SpecificStorage.GeomNames = "domain"
 ifield.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-ifield.Phase.Names = 'water'
+ifield.Phase.Names = "water"
 
-ifield.Phase.water.Density.Type = 'Constant'
+ifield.Phase.water.Density.Type = "Constant"
 ifield.Phase.water.Density.Value = 1.0
 
-ifield.Phase.water.Viscosity.Type = 'Constant'
+ifield.Phase.water.Viscosity.Type = "Constant"
 ifield.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-ifield.Contaminants.Names = ''
+ifield.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-ifield.Geom.Retardation.GeomNames = ''
+ifield.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 ifield.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 ifield.TimingInfo.BaseUnit = 1.0
 ifield.TimingInfo.StartCount = 0
 ifield.TimingInfo.StartTime = 0.0
-ifield.TimingInfo.StopTime = 30.0*1
+ifield.TimingInfo.StopTime = 30.0 * 1
 ifield.TimingInfo.DumpInterval = 0
-ifield.TimeStep.Type = 'Constant'
+ifield.TimeStep.Type = "Constant"
 ifield.TimeStep.Value = 10.0
 ifield.TimingInfo.DumpAtEnd = True
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-ifield.Geom.Porosity.GeomNames = 'domain'
-ifield.Geom.domain.Porosity.Type = 'Constant'
+ifield.Geom.Porosity.GeomNames = "domain"
+ifield.Geom.domain.Porosity.Type = "Constant"
 ifield.Geom.domain.Porosity.Value = 0.3680
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-ifield.Domain.GeomName = 'domain'
+ifield.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-ifield.Phase.RelPerm.Type = 'VanGenuchten'
+ifield.Phase.RelPerm.Type = "VanGenuchten"
 
 if useIndicatorField == 1:
-  ifield.Phase.RelPerm.GeomNames = 'indicator'
-  ifield.Geom.indicator.RelPerm.Alpha = 3.34
-  ifield.Geom.indicator.RelPerm.N = 1.982
+    ifield.Phase.RelPerm.GeomNames = "indicator"
+    ifield.Geom.indicator.RelPerm.Alpha = 3.34
+    ifield.Geom.indicator.RelPerm.N = 1.982
 else:
-  ifield.Phase.RelPerm.GeomNames = 'domain'
-  ifield.Geom.domain.RelPerm.Alpha = 3.34
-  ifield.Geom.domain.RelPerm.N = 1.982
+    ifield.Phase.RelPerm.GeomNames = "domain"
+    ifield.Geom.domain.RelPerm.Alpha = 3.34
+    ifield.Geom.domain.RelPerm.N = 1.982
 
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-ifield.Phase.Saturation.Type = 'VanGenuchten'
-ifield.Phase.Saturation.GeomNames = 'domain'
+ifield.Phase.Saturation.Type = "VanGenuchten"
+ifield.Phase.Saturation.GeomNames = "domain"
 
 ifield.Geom.domain.Saturation.Alpha = 3.34
 ifield.Geom.domain.Saturation.N = 1.982
 ifield.Geom.domain.Saturation.SRes = 0.2771
 ifield.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-ifield.Wells.Names = ''
+ifield.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-ifield.Cycle.Names = 'constant onoff'
-ifield.Cycle.constant.Names = 'alltime'
+ifield.Cycle.Names = "constant onoff"
+ifield.Cycle.constant.Names = "alltime"
 ifield.Cycle.constant.alltime.Length = 1
 ifield.Cycle.constant.Repeat = -1
 
-ifield.Cycle.onoff.Names = 'on off'
+ifield.Cycle.onoff.Names = "on off"
 ifield.Cycle.onoff.on.Length = 10
 ifield.Cycle.onoff.off.Length = 90
 ifield.Cycle.onoff.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 ifield.BCPressure.PatchNames = ifield.Geom.domain.Patches
 
-ifield.Patch.infiltration.BCPressure.Type = 'FluxConst'
-ifield.Patch.infiltration.BCPressure.Cycle = 'constant'
+ifield.Patch.infiltration.BCPressure.Type = "FluxConst"
+ifield.Patch.infiltration.BCPressure.Cycle = "constant"
 ifield.Patch.infiltration.BCPressure.alltime.Value = -0.10
 ifield.Patch.infiltration.BCPressure.off.Value = 0.0
 
-ifield.Patch.x_lower.BCPressure.Type = 'FluxConst'
-ifield.Patch.x_lower.BCPressure.Cycle = 'constant'
+ifield.Patch.x_lower.BCPressure.Type = "FluxConst"
+ifield.Patch.x_lower.BCPressure.Cycle = "constant"
 ifield.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-ifield.Patch.y_lower.BCPressure.Type = 'FluxConst'
-ifield.Patch.y_lower.BCPressure.Cycle = 'constant'
+ifield.Patch.y_lower.BCPressure.Type = "FluxConst"
+ifield.Patch.y_lower.BCPressure.Cycle = "constant"
 ifield.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-ifield.Patch.z_lower.BCPressure.Type = 'FluxConst'
-ifield.Patch.z_lower.BCPressure.Cycle = 'constant'
+ifield.Patch.z_lower.BCPressure.Type = "FluxConst"
+ifield.Patch.z_lower.BCPressure.Cycle = "constant"
 ifield.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-ifield.Patch.x_upper.BCPressure.Type = 'FluxConst'
-ifield.Patch.x_upper.BCPressure.Cycle = 'constant'
+ifield.Patch.x_upper.BCPressure.Type = "FluxConst"
+ifield.Patch.x_upper.BCPressure.Cycle = "constant"
 ifield.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-ifield.Patch.y_upper.BCPressure.Type = 'FluxConst'
-ifield.Patch.y_upper.BCPressure.Cycle = 'constant'
+ifield.Patch.y_upper.BCPressure.Type = "FluxConst"
+ifield.Patch.y_upper.BCPressure.Cycle = "constant"
 ifield.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-ifield.Patch.z_upper.BCPressure.Type = 'FluxConst'
-ifield.Patch.z_upper.BCPressure.Cycle = 'constant'
+ifield.Patch.z_upper.BCPressure.Type = "FluxConst"
+ifield.Patch.z_upper.BCPressure.Cycle = "constant"
 ifield.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-ifield.TopoSlopesX.Type = 'Constant'
-ifield.TopoSlopesX.GeomNames = ''
+ifield.TopoSlopesX.Type = "Constant"
+ifield.TopoSlopesX.GeomNames = ""
 ifield.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-ifield.TopoSlopesY.Type = 'Constant'
-ifield.TopoSlopesY.GeomNames = ''
+ifield.TopoSlopesY.Type = "Constant"
+ifield.TopoSlopesY.GeomNames = ""
 ifield.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-ifield.Mannings.Type = 'Constant'
-ifield.Mannings.GeomNames = ''
+ifield.Mannings.Type = "Constant"
+ifield.Mannings.GeomNames = ""
 ifield.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-ifield.ICPressure.Type = 'HydroStaticPatch'
-ifield.ICPressure.GeomNames = 'domain'
+ifield.ICPressure.Type = "HydroStaticPatch"
+ifield.ICPressure.GeomNames = "domain"
 
 ifield.Geom.domain.ICPressure.Value = 1.0
-ifield.Geom.domain.ICPressure.RefPatch = 'z_lower'
-ifield.Geom.domain.ICPressure.RefGeom = 'domain'
+ifield.Geom.domain.ICPressure.RefPatch = "z_lower"
+ifield.Geom.domain.ICPressure.RefGeom = "domain"
 
 ifield.Geom.infiltration.ICPressure.Value = 10.0
-ifield.Geom.infiltration.ICPressure.RefPatch = 'infiltration'
-ifield.Geom.infiltration.ICPressure.RefGeom = 'domain'
+ifield.Geom.infiltration.ICPressure.RefPatch = "infiltration"
+ifield.Geom.infiltration.ICPressure.RefGeom = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-ifield.PhaseSources.water.Type = 'Constant'
-ifield.PhaseSources.water.GeomNames = 'domain'
+ifield.PhaseSources.water.Type = "Constant"
+ifield.PhaseSources.water.GeomNames = "domain"
 ifield.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-ifield.KnownSolution = 'NoKnownSolution'
+ifield.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-ifield.Solver = 'Richards'
+ifield.Solver = "Richards"
 ifield.Solver.MaxIter = 1
 
 ifield.Solver.Nonlinear.MaxIter = 15
@@ -313,36 +316,52 @@
 ifield.Solver.Linear.KrylovDimension = 25
 ifield.Solver.Linear.MaxRestarts = 2
 
-ifield.Solver.Linear.Preconditioner = 'MGSemi'
+ifield.Solver.Linear.Preconditioner = "MGSemi"
 ifield.Solver.Linear.Preconditioner.MGSemi.MaxIter = 1
 ifield.Solver.Linear.Preconditioner.MGSemi.MaxLevels = 100
 
-ifield.Solver.PrintSubsurfData = False
-ifield.Solver.PrintPressure = False
-ifield.Solver.PrintSaturation = False
-ifield.Solver.PrintConcentration = False
-
-ifield.Solver.WriteSiloSubsurfData = True
-ifield.Solver.WriteSiloPressure = True
-ifield.Solver.WriteSiloSaturation = True
-ifield.Solver.WriteSiloConcentration = True
-
-#-----------------------------------------------------------------------------
-# Ensure generated pfidb match our expectation
-#-----------------------------------------------------------------------------
-
-generatedFile, runId = ifield.write()
-
-# Prevent regression
-with open(generatedFile) as new, open(get_absolute_path('$PF_SRC/test/correct_output/indicator_field.pfidb.ref')) as ref:
-  if new.read() != ref.read():
-    print('Generated PFIDB does not match our expectation')
-    sys.exit(1)
-
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-ifield.dist('small_domain_indicator_field.pfb')
-
-ifield.run()
+# -----------------------------------------------------------------------------
+
+ifield.dist(dir_name + "/small_domain_indicator_field.pfb")
+correct_output_dir_name = get_absolute_path("../correct_output")
+ifield.run(working_directory=dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z", "porosity"]
+
+
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+for i in range(2):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/key_validation/key_initialization_order.py b/test/python/key_validation/key_initialization_order.py
index 27ce7b087..951c14461 100644
--- a/test/python/key_validation/key_initialization_order.py
+++ b/test/python/key_validation/key_initialization_order.py
@@ -480,13 +480,13 @@ def set_value(self):
         self.run.Wells.well1.alltime.Concentration.water.contaminant1.Value = 0.0
 
     def test_well_first(self):
-        """ Should be able to set Wells[".{well_name}"][".{interval_name}"].Concentration[".{phase_name}"][".{contaminant_name}"] setting Wells.Names before Contaminants.Names """
+        """Should be able to set Wells[".{well_name}"][".{interval_name}"].Concentration[".{phase_name}"][".{contaminant_name}"] setting Wells.Names before Contaminants.Names"""
         self.set_up_wells()
         self.set_up_contaminants()
         self.set_value()
 
     def test_contam_first(self):
-        """ Should be able to set Wells[".{well_name}"][".{interval_name}"].Concentration[".{phase_name}"][".{contaminant_name}"] setting Contaminants.Names before Wells.Names """
+        """Should be able to set Wells[".{well_name}"][".{interval_name}"].Concentration[".{phase_name}"][".{contaminant_name}"] setting Contaminants.Names before Wells.Names"""
         self.set_up_contaminants()
         self.set_up_wells()
         self.set_value()
@@ -500,13 +500,13 @@ def set_value(self):
         self.run.Patch.x_lower.BCSaturation.water.Value = 2.0
 
     def test_patch_first(self):
-        """ Should be able to set json.Patch[".{patch_name}"].BCSaturation[".{phase_name}"].Value """
+        """Should be able to set json.Patch[".{patch_name}"].BCSaturation[".{phase_name}"].Value"""
         self.set_up_patches()
         self.set_up_phases()
         self.set_value()
 
     def test_phase_first(self):
-        """ Should be able to set json.Patch[".{patch_name}"].BCSaturation[".{phase_name}"].Value """
+        """Should be able to set json.Patch[".{patch_name}"].BCSaturation[".{phase_name}"].Value"""
         self.set_up_phases()
         self.set_up_patches()
         self.set_value()
@@ -520,13 +520,13 @@ def set_value(self):
         self.run.Patch.x_lower.BCPressure.alltime.Value = 2.0
 
     def test_patch_first(self):
-        """ Should be able to set json.Patch[".{patch_name}"].BCPressure[".{interval_name}"].Value """
+        """Should be able to set json.Patch[".{patch_name}"].BCPressure[".{interval_name}"].Value"""
         self.set_up_patches()
         self.set_up_intervals()
         self.set_value()
 
     def test_interval_first(self):
-        """ Should be able to set json.Patch[".{patch_name}"].BCPressure[".{interval_name}"].Value """
+        """Should be able to set json.Patch[".{patch_name}"].BCPressure[".{interval_name}"].Value"""
         self.set_up_intervals()
         self.set_up_patches()
         self.set_value()
@@ -542,42 +542,42 @@ def set_int_value(self):
         )
 
     def test_patch_interval_phase_order(self):
-        """ Should be able to set json.Patch[".{patch_name}"].BCPressure[".{interval_name}"][".{phase_name}"].IntValue """
+        """Should be able to set json.Patch[".{patch_name}"].BCPressure[".{interval_name}"][".{phase_name}"].IntValue"""
         self.set_up_patches()
         self.set_up_intervals()
         self.set_up_phases()
         self.set_int_value()
 
     def test_patch_phase_interval_order(self):
-        """ Should be able to set json.Patch[".{patch_name}"].BCPressure[".{interval_name}"][".{phase_name}"].IntValue   """
+        """Should be able to set json.Patch[".{patch_name}"].BCPressure[".{interval_name}"][".{phase_name}"].IntValue"""
         self.set_up_patches()
         self.set_up_phases()
         self.set_up_intervals()
         self.set_int_value()
 
     def test_interval_patch_phase_order(self):
-        """ Should be able to set json.Patch[".{patch_name}"].BCPressure[".{interval_name}"][".{phase_name}"].IntValue   """
+        """Should be able to set json.Patch[".{patch_name}"].BCPressure[".{interval_name}"][".{phase_name}"].IntValue"""
         self.set_up_intervals()
         self.set_up_patches()
         self.set_up_phases()
         self.set_int_value()
 
     def test_interval_phase_patch_order(self):
-        """ Should be able to set json.Patch[".{patch_name}"].BCPressure[".{interval_name}"][".{phase_name}"].IntValue """
+        """Should be able to set json.Patch[".{patch_name}"].BCPressure[".{interval_name}"][".{phase_name}"].IntValue"""
         self.set_up_intervals()
         self.set_up_phases()
         self.set_up_patches()
         self.set_int_value()
 
     def test_phase_patch_interval_order(self):
-        """ Should be able to set json.Patch[".{patch_name}"].BCPressure[".{interval_name}"][".{phase_name}"].IntValue   """
+        """Should be able to set json.Patch[".{patch_name}"].BCPressure[".{interval_name}"][".{phase_name}"].IntValue"""
         self.set_up_phases()
         self.set_up_patches()
         self.set_up_intervals()
         self.set_int_value()
 
     def test_phase_interval_patch_order(self):
-        """ Should be able to set json.Patch[".{patch_name}"].BCPressure[".{interval_name}"][".{phase_name}"].IntValue   """
+        """Should be able to set json.Patch[".{patch_name}"].BCPressure[".{interval_name}"][".{phase_name}"].IntValue"""
         self.set_up_phases()
         self.set_up_intervals()
         self.set_up_patches()
diff --git a/test/python/new_features/CMakeLists.txt b/test/python/new_features/CMakeLists.txt
index 87c02ebd9..d1c99c010 100644
--- a/test/python/new_features/CMakeLists.txt
+++ b/test/python/new_features/CMakeLists.txt
@@ -17,6 +17,7 @@ set(TESTS
   normalize_location
   os_function
   # pfb_mask
+  pfb_reader
   pfidb_pfset
   pfset_test
   selection_methods
diff --git a/test/python/new_features/LW_table_domain.py b/test/python/new_features/LW_table_domain.py
index 462d15ac0..ab5bb0483 100644
--- a/test/python/new_features/LW_table_domain.py
+++ b/test/python/new_features/LW_table_domain.py
@@ -1,7 +1,7 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # SCRIPT TO RUN LITTLE WASHITA DOMAIN WITH TERRAIN-FOLLOWING GRID
 # Using combination of tables and domain builder
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 import sys
 
@@ -13,17 +13,17 @@
 
 LW_Test.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set Processor topology
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_Test.Process.Topology.P = 1
 LW_Test.Process.Topology.Q = 1
 LW_Test.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_Test.ComputationalGrid.Lower.X = 0.0
 LW_Test.ComputationalGrid.Lower.Y = 0.0
@@ -37,69 +37,70 @@
 LW_Test.ComputationalGrid.NY = 41
 LW_Test.ComputationalGrid.NZ = 50
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Timing (time units is set by units of permeability)
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_Test.TimingInfo.BaseUnit = 1.0
 LW_Test.TimingInfo.StartCount = 0
 LW_Test.TimingInfo.StartTime = 0.0
 LW_Test.TimingInfo.StopTime = 1000.0
 LW_Test.TimingInfo.DumpInterval = 1.0
-LW_Test.TimeStep.Type = 'Constant'
+LW_Test.TimeStep.Type = "Constant"
 LW_Test.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.Cycle.Names = 'constant rainrec'
-LW_Test.Cycle.constant.Names = 'alltime'
+LW_Test.Cycle.Names = "constant rainrec"
+LW_Test.Cycle.constant.Names = "alltime"
 LW_Test.Cycle.constant.alltime.Length = 1
 LW_Test.Cycle.constant.Repeat = -1
 
-LW_Test.Cycle.rainrec.Names = 'rain rec'
+LW_Test.Cycle.rainrec.Names = "rain rec"
 LW_Test.Cycle.rainrec.rain.Length = 10
 LW_Test.Cycle.rainrec.rec.Length = 150
 LW_Test.Cycle.rainrec.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Using domain builder
-#-----------------------------------------------------------------------------
-
-bounds = [
-    0.0, 41000.0,
-    0.0, 41000.0,
-    0.0, 100.0
-]
-
-domain_patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
-zero_flux_patches = 'x_lower x_upper y_lower y_upper z_lower'
-
-DomainBuilder(LW_Test) \
-    .no_wells() \
-    .no_contaminants() \
-    .water('domain') \
-    .variably_saturated() \
-    .box_domain('box_input', 'domain', bounds, domain_patches) \
-    .homogeneous_subsurface('domain', specific_storage=1.0e-5, isotropic=True) \
-    .zero_flux(zero_flux_patches, 'constant', 'alltime') \
-    .slopes_mannings('domain', slope_x='LW.slopex.pfb', slope_y='LW.slopey.pfb', mannings=5.52e-6) \
-    .ic_pressure('domain', patch='z_upper', pressure='press.init.pfb')
-
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+
+bounds = [0.0, 41000.0, 0.0, 41000.0, 0.0, 100.0]
+
+domain_patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
+zero_flux_patches = "x_lower x_upper y_lower y_upper z_lower"
+
+DomainBuilder(LW_Test).no_wells().no_contaminants().water(
+    "domain"
+).variably_saturated().box_domain(
+    "box_input", "domain", bounds, domain_patches
+).homogeneous_subsurface(
+    "domain", specific_storage=1.0e-5, isotropic=True
+).zero_flux(
+    zero_flux_patches, "constant", "alltime"
+).slopes_mannings(
+    "domain", slope_x="LW.slopex.pfb", slope_y="LW.slopey.pfb", mannings=5.52e-6
+).ic_pressure(
+    "domain", patch="z_upper", pressure="press.init.pfb"
+)
+
+# -----------------------------------------------------------------------------
 # Adding indi_input to GeomInput.Names
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.GeomInput.Names = 'box_input indi_input'
+LW_Test.GeomInput.Names = "box_input indi_input"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Indicator Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.GeomInput.indi_input.InputType = 'IndicatorField'
-LW_Test.GeomInput.indi_input.GeomNames = 's1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 g1 g2 g3 g4 g5 g6 g7 g8'
-LW_Test.Geom.indi_input.FileName = 'IndicatorFile_Gleeson.50z.pfb'
+LW_Test.GeomInput.indi_input.InputType = "IndicatorField"
+LW_Test.GeomInput.indi_input.GeomNames = (
+    "s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 g1 g2 g3 g4 g5 g6 g7 g8"
+)
+LW_Test.Geom.indi_input.FileName = "IndicatorFile_Gleeson.50z.pfb"
 
 LW_Test.GeomInput.s1.Value = 1
 LW_Test.GeomInput.s2.Value = 2
@@ -123,11 +124,11 @@
 LW_Test.GeomInput.g7.Value = 27
 LW_Test.GeomInput.g8.Value = 28
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Permeability (values in m/hr)
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_subsurface_properties = '''
+LW_subsurface_properties = """
 key     Perm          Porosity   RelPermAlpha  RelPermN  SatAlpha  SatN    SatSRes    SatSSat
 domain  0.2           0.4        3.5           2         3.5       2       0.2        1
 s1      0.269022595   0.375      3.548         4.162     3.548     4.162   0.000001   1
@@ -143,25 +144,23 @@
 g3      0.059         -          -             -         -         -       -          -
 g6      0.2           -          -             -         -         -       -          -
 g8      0.68          -          -             -         -         -       -          -
-'''
+"""
 
 # Setting subsurface properties
-SubsurfacePropertiesBuilder(LW_Test)\
-  .load_txt_content(LW_subsurface_properties) \
-  .apply()
+SubsurfacePropertiesBuilder(LW_Test).load_txt_content(LW_subsurface_properties).apply()
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_Test.BCPressure.PatchNames = LW_Test.Geom.domain.Patches
 
-LW_Test.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-LW_Test.Patch.z_upper.BCPressure.Cycle = 'rainrec'
+LW_Test.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+LW_Test.Patch.z_upper.BCPressure.Cycle = "rainrec"
 LW_Test.Patch.z_upper.BCPressure.rain.Value = -0.1
 LW_Test.Patch.z_upper.BCPressure.rec.Value = 0.0000
 
-#----------------------------------------------------------------
+# ----------------------------------------------------------------
 # Outputs
 # ---------------------------------------------------------------
 
@@ -170,22 +169,22 @@
 LW_Test.Solver.PrintSaturation = True
 LW_Test.Solver.PrintMask = True
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.KnownSolution = 'NoKnownSolution'
+LW_Test.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_Test.Solver.TerrainFollowingGrid = True
 LW_Test.Solver.Nonlinear.VariableDz = False
 
 LW_Test.Solver.MaxIter = 25000
-LW_Test.Solver.Drop = 1E-20
-LW_Test.Solver.AbsTol = 1E-8
+LW_Test.Solver.Drop = 1e-20
+LW_Test.Solver.AbsTol = 1e-8
 LW_Test.Solver.MaxConvergenceFailures = 8
 LW_Test.Solver.Nonlinear.MaxIter = 80
 LW_Test.Solver.Nonlinear.ResidualTol = 1e-6
@@ -193,31 +192,33 @@
 LW_Test.Solver.Nonlinear.EtaValue = 0.001
 LW_Test.Solver.Nonlinear.DerivativeEpsilon = 1e-16
 LW_Test.Solver.Nonlinear.StepTol = 1e-30
-LW_Test.Solver.Nonlinear.Globalization = 'LineSearch'
+LW_Test.Solver.Nonlinear.Globalization = "LineSearch"
 LW_Test.Solver.Linear.KrylovDimension = 70
 LW_Test.Solver.Linear.MaxRestarts = 2
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Test validation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+
 
 def test_output(file_name):
-  LW_Test.write(file_name, file_format='yaml')
-  with open(get_absolute_path(f'{file_name}.yaml')) as new, \
-      open(get_absolute_path('$PF_SRC/test/correct_output/LW_test_ref.yaml.ref')) as ref:
-    if new.read() == ref.read():
-      print('Success we have the same file')
-      return True
-    else:
-      print('Files are different')
-      return False
+    LW_Test.write(file_name, file_format="yaml")
+    with open(get_absolute_path(f"{file_name}.yaml")) as new, open(
+        get_absolute_path("$PF_SRC/test/correct_output/LW_test_ref.yaml.ref")
+    ) as ref:
+        if new.read() == ref.read():
+            print("Success we have the same file")
+            return True
+        else:
+            print("Files are different")
+            return False
+
 
 # testing output
-print('+'*40)
-print('Comparing table and domain builder output to ref:')
-print('...')
+print("+" * 40)
+print("Comparing table and domain builder output to ref:")
+print("...")
 # LW_Test.write_subsurface_table('inline_input.txt')
-if not test_output('LW_table_domain'):
-  sys.exit(1)
-print('')
-
+if not test_output("LW_table_domain"):
+    sys.exit(1)
+print("")
diff --git a/test/python/new_features/asc_write.py b/test/python/new_features/asc_write.py
index 56b278d62..6dd82ab8a 100644
--- a/test/python/new_features/asc_write.py
+++ b/test/python/new_features/asc_write.py
@@ -1,41 +1,51 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # example for pfsol generation
 # checking for correct .asc file writing
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 import sys
 
 from parflow import Run
 from parflow.tools.fs import get_absolute_path
-from parflow.tools.io import load_patch_matrix_from_sa_file, load_patch_matrix_from_asc_file, write_patch_matrix_as_sa
+from parflow.tools.io import (
+    load_patch_matrix_from_sa_file,
+    load_patch_matrix_from_asc_file,
+    write_patch_matrix_as_sa,
+)
 
 asc_write = Run("asc_write", __file__)
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-asc_write.GeomInput.Names = 'domaininput'
+asc_write.GeomInput.Names = "domaininput"
 
-asc_write.GeomInput.domaininput.GeomName = 'domain'
-asc_write.GeomInput.domaininput.GeomNames = 'domain'
+asc_write.GeomInput.domaininput.GeomName = "domain"
+asc_write.GeomInput.domaininput.GeomNames = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # PFSOL generation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino_mask = load_patch_matrix_from_sa_file(get_absolute_path('$PF_SRC/test/input/Sabino_Mask.sa'))
+sabino_mask = load_patch_matrix_from_sa_file(
+    get_absolute_path("$PF_SRC/test/input/Sabino_Mask.sa")
+)
 # sabino_mask = load_patch_matrix_from_asc_file(get_absolute_path('Sabino_Mask.asc'))
 
-sabino_mask_written = write_patch_matrix_as_sa(sabino_mask,
-                                               get_absolute_path('$PF_SRC/test/python/new_features/sabino_mask_written.sa'))
+sabino_mask_written = write_patch_matrix_as_sa(
+    sabino_mask,
+    get_absolute_path("$PF_SRC/test/python/new_features/sabino_mask_written.sa"),
+)
 
-with open(get_absolute_path('$PF_SRC/test/input/Sabino_Mask.sa'), "rt") as ref:
-    with open(get_absolute_path('$PF_SRC/test/python/new_features/sabino_mask_written.sa'), "rt") as new:
+with open(get_absolute_path("$PF_SRC/test/input/Sabino_Mask.sa"), "rt") as ref:
+    with open(
+        get_absolute_path("$PF_SRC/test/python/new_features/sabino_mask_written.sa"),
+        "rt",
+    ) as new:
         if new.read() == ref.read():
-            print('Success we have the same file')
+            print("Success we have the same file")
         else:
-            print('Files are different')
+            print("Files are different")
             sys.exit(1)
-
diff --git a/test/python/new_features/clm/washita.py b/test/python/new_features/clm/washita.py
index 8b390724f..8cf18d7fb 100644
--- a/test/python/new_features/clm/washita.py
+++ b/test/python/new_features/clm/washita.py
@@ -1,18 +1,18 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # SCRIPT TO RUN LITTLE WASHITA DOMAIN WITH TERRAIN-FOLLOWING GRID
 # DETAILS
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 from parflow import Run
 from parflow.tools.fs import cp, get_absolute_path, rm
 from parflow.tools.builders import CLMImporter
 
 
-washita = Run('washita', __file__)
+washita = Run("washita", __file__)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Import the driver files, and remove them.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 # Set up the computational grid
 washita.ComputationalGrid.NX = 41
@@ -20,67 +20,65 @@
 washita.ComputationalGrid.NZ = 50
 
 # Copy over the driver files
-cp('$PF_SRC/test/tcl/washita/clm_input/drv_clmin.dat')
-cp('$PF_SRC/test/tcl/washita/clm_input/drv_vegm.alluv.dat')
-cp('$PF_SRC/test/tcl/washita/clm_input/drv_vegp.dat')
+cp("$PF_SRC/test/tcl/washita/clm_input/drv_clmin.dat")
+cp("$PF_SRC/test/tcl/washita/clm_input/drv_vegm.alluv.dat")
+cp("$PF_SRC/test/tcl/washita/clm_input/drv_vegp.dat")
 
 # Import the driver files
-CLMImporter(washita) \
-    .set_default_land_names() \
-    .input_file() \
-    .parameters_file() \
-    .map_file('drv_vegm.alluv.dat')
+CLMImporter(washita).set_default_land_names().input_file().parameters_file().map_file(
+    "drv_vegm.alluv.dat"
+)
 
 # Write out the yaml file
-washita.write('washita', file_format='yaml')
+washita.write("washita", file_format="yaml")
 
 # Remove the driver files
-rm('drv_clmin.dat')
-rm('drv_vegm.alluv.dat')
-rm('drv_vegp.dat')
+rm("drv_clmin.dat")
+rm("drv_vegm.alluv.dat")
+rm("drv_vegp.dat")
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Make a directory for the simulation run and copy files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dir_name = get_absolute_path('.')
+dir_name = get_absolute_path(".")
 
-cp('$PF_SRC/test/tcl/washita/parflow_input/LW.slopex.pfb', dir_name)
-cp('$PF_SRC/test/tcl/washita/parflow_input/LW.slopey.pfb', dir_name)
-cp('$PF_SRC/test/tcl/washita/parflow_input/IndicatorFile_Gleeson.50z.pfb', dir_name)
-cp('$PF_SRC/test/tcl/washita/parflow_input/press.init.pfb', dir_name)
+cp("$PF_SRC/test/tcl/washita/parflow_input/LW.slopex.pfb", dir_name)
+cp("$PF_SRC/test/tcl/washita/parflow_input/LW.slopey.pfb", dir_name)
+cp("$PF_SRC/test/tcl/washita/parflow_input/IndicatorFile_Gleeson.50z.pfb", dir_name)
+cp("$PF_SRC/test/tcl/washita/parflow_input/press.init.pfb", dir_name)
 
 
 nldas_files = [
-  'NLDAS.DSWR.000001_to_000024.pfb',
-  'NLDAS.DLWR.000001_to_000024.pfb',
-  'NLDAS.APCP.000001_to_000024.pfb',
-  'NLDAS.Temp.000001_to_000024.pfb',
-  'NLDAS.UGRD.000001_to_000024.pfb',
-  'NLDAS.VGRD.000001_to_000024.pfb',
-  'NLDAS.Press.000001_to_000024.pfb',
-  'NLDAS.SPFH.000001_to_000024.pfb'
-  ]
+    "NLDAS.DSWR.000001_to_000024.pfb",
+    "NLDAS.DLWR.000001_to_000024.pfb",
+    "NLDAS.APCP.000001_to_000024.pfb",
+    "NLDAS.Temp.000001_to_000024.pfb",
+    "NLDAS.UGRD.000001_to_000024.pfb",
+    "NLDAS.VGRD.000001_to_000024.pfb",
+    "NLDAS.Press.000001_to_000024.pfb",
+    "NLDAS.SPFH.000001_to_000024.pfb",
+]
 
 for file in nldas_files:
-  cp('$PF_SRC/test/tcl/washita/NLDAS/' + file, dir_name)
-  washita.dist(dir_name + '/' + file, R=24)
+    cp("$PF_SRC/test/tcl/washita/NLDAS/" + file, dir_name)
+    washita.dist(dir_name + "/" + file, R=24)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 washita.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set Processor topology
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 washita.Process.Topology.P = 1
 washita.Process.Topology.Q = 1
 washita.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 washita.ComputationalGrid.Lower.X = 0.0
 washita.ComputationalGrid.Lower.Y = 0.0
@@ -92,22 +90,22 @@
 
 # NX, NY, and NZ already set above
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.GeomInput.Names = 'box_input indi_input'
+washita.GeomInput.Names = "box_input indi_input"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.GeomInput.box_input.InputType = 'Box'
-washita.GeomInput.box_input.GeomName = 'domain'
+washita.GeomInput.box_input.InputType = "Box"
+washita.GeomInput.box_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 washita.Geom.domain.Lower.X = 0.0
 washita.Geom.domain.Lower.Y = 0.0
@@ -116,15 +114,17 @@
 washita.Geom.domain.Upper.X = 41000.0
 washita.Geom.domain.Upper.Y = 41000.0
 washita.Geom.domain.Upper.Z = 100.0
-washita.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+washita.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Indicator Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.GeomInput.indi_input.InputType = 'IndicatorField'
-washita.GeomInput.indi_input.GeomNames = 's1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 g1 g2 g3 g4 g5 g6 g7 g8'
-washita.Geom.indi_input.FileName = 'IndicatorFile_Gleeson.50z.pfb'
+washita.GeomInput.indi_input.InputType = "IndicatorField"
+washita.GeomInput.indi_input.GeomNames = (
+    "s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 g1 g2 g3 g4 g5 g6 g7 g8"
+)
+washita.Geom.indi_input.FileName = "IndicatorFile_Gleeson.50z.pfb"
 
 washita.GeomInput.s1.Value = 1
 washita.GeomInput.s2.Value = 2
@@ -148,226 +148,226 @@
 washita.GeomInput.g7.Value = 27
 washita.GeomInput.g8.Value = 28
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Permeability (values in m/hr)
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Geom.Perm.Names = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9 g2 g3 g6 g8'
+washita.Geom.Perm.Names = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9 g2 g3 g6 g8"
 
-washita.Geom.domain.Perm.Type = 'Constant'
+washita.Geom.domain.Perm.Type = "Constant"
 washita.Geom.domain.Perm.Value = 0.2
 
-washita.Geom.s1.Perm.Type = 'Constant'
+washita.Geom.s1.Perm.Type = "Constant"
 washita.Geom.s1.Perm.Value = 0.269022595
 
-washita.Geom.s2.Perm.Type = 'Constant'
+washita.Geom.s2.Perm.Type = "Constant"
 washita.Geom.s2.Perm.Value = 0.043630356
 
-washita.Geom.s3.Perm.Type = 'Constant'
+washita.Geom.s3.Perm.Type = "Constant"
 washita.Geom.s3.Perm.Value = 0.015841225
 
-washita.Geom.s4.Perm.Type = 'Constant'
+washita.Geom.s4.Perm.Type = "Constant"
 washita.Geom.s4.Perm.Value = 0.007582087
 
-washita.Geom.s5.Perm.Type = 'Constant'
+washita.Geom.s5.Perm.Type = "Constant"
 washita.Geom.s5.Perm.Value = 0.01818816
 
-washita.Geom.s6.Perm.Type = 'Constant'
+washita.Geom.s6.Perm.Type = "Constant"
 washita.Geom.s6.Perm.Value = 0.005009435
 
-washita.Geom.s7.Perm.Type = 'Constant'
+washita.Geom.s7.Perm.Type = "Constant"
 washita.Geom.s7.Perm.Value = 0.005492736
 
-washita.Geom.s8.Perm.Type = 'Constant'
+washita.Geom.s8.Perm.Type = "Constant"
 washita.Geom.s8.Perm.Value = 0.004675077
 
-washita.Geom.s9.Perm.Type = 'Constant'
+washita.Geom.s9.Perm.Type = "Constant"
 washita.Geom.s9.Perm.Value = 0.003386794
 
-washita.Geom.g2.Perm.Type = 'Constant'
+washita.Geom.g2.Perm.Type = "Constant"
 washita.Geom.g2.Perm.Value = 0.025
 
-washita.Geom.g3.Perm.Type = 'Constant'
+washita.Geom.g3.Perm.Type = "Constant"
 washita.Geom.g3.Perm.Value = 0.059
 
-washita.Geom.g6.Perm.Type = 'Constant'
+washita.Geom.g6.Perm.Type = "Constant"
 washita.Geom.g6.Perm.Value = 0.2
 
-washita.Geom.g8.Perm.Type = 'Constant'
+washita.Geom.g8.Perm.Type = "Constant"
 washita.Geom.g8.Perm.Value = 0.68
 
-washita.Perm.TensorType = 'TensorByGeom'
-washita.Geom.Perm.TensorByGeom.Names = 'domain'
+washita.Perm.TensorType = "TensorByGeom"
+washita.Geom.Perm.TensorByGeom.Names = "domain"
 washita.Geom.domain.Perm.TensorValX = 1.0
 washita.Geom.domain.Perm.TensorValY = 1.0
 washita.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.SpecificStorage.Type = 'Constant'
-washita.SpecificStorage.GeomNames = 'domain'
+washita.SpecificStorage.Type = "Constant"
+washita.SpecificStorage.GeomNames = "domain"
 washita.Geom.domain.SpecificStorage.Value = 1.0e-5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Phase.Names = 'water'
-washita.Phase.water.Density.Type = 'Constant'
+washita.Phase.Names = "water"
+washita.Phase.water.Density.Type = "Constant"
 washita.Phase.water.Density.Value = 1.0
-washita.Phase.water.Viscosity.Type = 'Constant'
+washita.Phase.water.Viscosity.Type = "Constant"
 washita.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Contaminants.Names = ''
+washita.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 washita.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Timing (time units is set by units of permeability)
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 washita.TimingInfo.BaseUnit = 1.0
 washita.TimingInfo.StartCount = 0
 washita.TimingInfo.StartTime = 0.0
 washita.TimingInfo.StopTime = 12.0
 washita.TimingInfo.DumpInterval = 24.0
-washita.TimeStep.Type = 'Constant'
+washita.TimeStep.Type = "Constant"
 washita.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Geom.Porosity.GeomNames = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9'
+washita.Geom.Porosity.GeomNames = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9"
 
-washita.Geom.domain.Porosity.Type = 'Constant'
+washita.Geom.domain.Porosity.Type = "Constant"
 washita.Geom.domain.Porosity.Value = 0.4
 
-washita.Geom.s1.Porosity.Type = 'Constant'
+washita.Geom.s1.Porosity.Type = "Constant"
 washita.Geom.s1.Porosity.Value = 0.375
 
-washita.Geom.s2.Porosity.Type = 'Constant'
+washita.Geom.s2.Porosity.Type = "Constant"
 washita.Geom.s2.Porosity.Value = 0.39
 
-washita.Geom.s3.Porosity.Type = 'Constant'
+washita.Geom.s3.Porosity.Type = "Constant"
 washita.Geom.s3.Porosity.Value = 0.387
 
-washita.Geom.s4.Porosity.Type = 'Constant'
+washita.Geom.s4.Porosity.Type = "Constant"
 washita.Geom.s4.Porosity.Value = 0.439
 
-washita.Geom.s5.Porosity.Type = 'Constant'
+washita.Geom.s5.Porosity.Type = "Constant"
 washita.Geom.s5.Porosity.Value = 0.489
 
-washita.Geom.s6.Porosity.Type = 'Constant'
+washita.Geom.s6.Porosity.Type = "Constant"
 washita.Geom.s6.Porosity.Value = 0.399
 
-washita.Geom.s7.Porosity.Type = 'Constant'
+washita.Geom.s7.Porosity.Type = "Constant"
 washita.Geom.s7.Porosity.Value = 0.384
 
-washita.Geom.s8.Porosity.Type = 'Constant'
+washita.Geom.s8.Porosity.Type = "Constant"
 washita.Geom.s8.Porosity.Value = 0.482
 
-washita.Geom.s9.Porosity.Type = 'Constant'
+washita.Geom.s9.Porosity.Type = "Constant"
 washita.Geom.s9.Porosity.Value = 0.442
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Domain.GeomName = 'domain'
+washita.Domain.GeomName = "domain"
 
-#----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 # Mobility
-#----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 
-washita.Phase.water.Mobility.Type = 'Constant'
+washita.Phase.water.Mobility.Type = "Constant"
 washita.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Wells.Names = ''
+washita.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Cycle.Names = 'constant'
-washita.Cycle.constant.Names = 'alltime'
+washita.Cycle.Names = "constant"
+washita.Cycle.constant.Names = "alltime"
 washita.Cycle.constant.alltime.Length = 1
 washita.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 washita.BCPressure.PatchNames = washita.Geom.domain.Patches
 
-washita.Patch.x_lower.BCPressure.Type = 'FluxConst'
-washita.Patch.x_lower.BCPressure.Cycle = 'constant'
+washita.Patch.x_lower.BCPressure.Type = "FluxConst"
+washita.Patch.x_lower.BCPressure.Cycle = "constant"
 washita.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-washita.Patch.y_lower.BCPressure.Type = 'FluxConst'
-washita.Patch.y_lower.BCPressure.Cycle = 'constant'
+washita.Patch.y_lower.BCPressure.Type = "FluxConst"
+washita.Patch.y_lower.BCPressure.Cycle = "constant"
 washita.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-washita.Patch.z_lower.BCPressure.Type = 'FluxConst'
-washita.Patch.z_lower.BCPressure.Cycle = 'constant'
+washita.Patch.z_lower.BCPressure.Type = "FluxConst"
+washita.Patch.z_lower.BCPressure.Cycle = "constant"
 washita.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-washita.Patch.x_upper.BCPressure.Type = 'FluxConst'
-washita.Patch.x_upper.BCPressure.Cycle = 'constant'
+washita.Patch.x_upper.BCPressure.Type = "FluxConst"
+washita.Patch.x_upper.BCPressure.Cycle = "constant"
 washita.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-washita.Patch.y_upper.BCPressure.Type = 'FluxConst'
-washita.Patch.y_upper.BCPressure.Cycle = 'constant'
+washita.Patch.y_upper.BCPressure.Type = "FluxConst"
+washita.Patch.y_upper.BCPressure.Cycle = "constant"
 washita.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-washita.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-washita.Patch.z_upper.BCPressure.Cycle = 'constant'
+washita.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+washita.Patch.z_upper.BCPressure.Cycle = "constant"
 washita.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Topo slopes in x-direction
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.TopoSlopesX.Type = 'PFBFile'
-washita.TopoSlopesX.GeomNames = 'domain'
-washita.TopoSlopesX.FileName = 'LW.slopex.pfb'
+washita.TopoSlopesX.Type = "PFBFile"
+washita.TopoSlopesX.GeomNames = "domain"
+washita.TopoSlopesX.FileName = "LW.slopex.pfb"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Topo slopes in y-direction
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.TopoSlopesY.Type = 'PFBFile'
-washita.TopoSlopesY.GeomNames = 'domain'
-washita.TopoSlopesY.FileName = 'LW.slopey.pfb'
+washita.TopoSlopesY.Type = "PFBFile"
+washita.TopoSlopesY.GeomNames = "domain"
+washita.TopoSlopesY.FileName = "LW.slopey.pfb"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mannings coefficient
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Mannings.Type = 'Constant'
-washita.Mannings.GeomNames = 'domain'
+washita.Mannings.Type = "Constant"
+washita.Mannings.GeomNames = "domain"
 washita.Mannings.Geom.domain.Value = 5.52e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Phase.RelPerm.Type = 'VanGenuchten'
-washita.Phase.RelPerm.GeomNames = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9 '
+washita.Phase.RelPerm.Type = "VanGenuchten"
+washita.Phase.RelPerm.GeomNames = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9 "
 
 washita.Geom.domain.RelPerm.Alpha = 3.5
 washita.Geom.domain.RelPerm.N = 2.0
@@ -399,15 +399,15 @@
 washita.Geom.s9.RelPerm.Alpha = 1.585
 washita.Geom.s9.RelPerm.N = 2.413
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Saturation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Phase.Saturation.Type = 'VanGenuchten'
-washita.Phase.Saturation.GeomNames = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9 '
+washita.Phase.Saturation.Type = "VanGenuchten"
+washita.Phase.Saturation.GeomNames = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9 "
 
 washita.Geom.domain.Saturation.Alpha = 3.5
-washita.Geom.domain.Saturation.N = 2.
+washita.Geom.domain.Saturation.N = 2.0
 washita.Geom.domain.Saturation.SRes = 0.2
 washita.Geom.domain.Saturation.SSat = 1.0
 
@@ -456,50 +456,50 @@
 washita.Geom.s9.Saturation.SRes = 0.000001
 washita.Geom.s9.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.PhaseSources.water.Type = 'Constant'
-washita.PhaseSources.water.GeomNames = 'domain'
+washita.PhaseSources.water.Type = "Constant"
+washita.PhaseSources.water.GeomNames = "domain"
 washita.PhaseSources.water.Geom.domain.Value = 0.0
 
-#----------------------------------------------------------------
+# ----------------------------------------------------------------
 # CLM Settings:
 # ------------------------------------------------------------
 
-washita.Solver.LSM = 'CLM'
-washita.Solver.CLM.CLMFileDir = '.'
+washita.Solver.LSM = "CLM"
+washita.Solver.CLM.CLMFileDir = "."
 washita.Solver.CLM.Print1dOut = False
 washita.Solver.CLM.CLMDumpInterval = 1
 
-washita.Solver.CLM.MetForcing = '3D'
-washita.Solver.CLM.MetFileName = 'NLDAS'
-washita.Solver.CLM.MetFilePath = '.'
+washita.Solver.CLM.MetForcing = "3D"
+washita.Solver.CLM.MetFileName = "NLDAS"
+washita.Solver.CLM.MetFilePath = "."
 washita.Solver.CLM.MetFileNT = 24
 washita.Solver.CLM.IstepStart = 1
 
-washita.Solver.CLM.EvapBeta = 'Linear'
-washita.Solver.CLM.VegWaterStress = 'Saturation'
+washita.Solver.CLM.EvapBeta = "Linear"
+washita.Solver.CLM.VegWaterStress = "Saturation"
 washita.Solver.CLM.ResSat = 0.1
 washita.Solver.CLM.WiltingPoint = 0.12
 washita.Solver.CLM.FieldCapacity = 0.98
-washita.Solver.CLM.IrrigationType = 'none'
+washita.Solver.CLM.IrrigationType = "none"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-washita.ICPressure.Type = 'PFBFile'
-washita.ICPressure.GeomNames = 'domain'
-washita.Geom.domain.ICPressure.RefPatch = 'z_upper'
-washita.Geom.domain.ICPressure.FileName = 'press.init.pfb'
+washita.ICPressure.Type = "PFBFile"
+washita.ICPressure.GeomNames = "domain"
+washita.Geom.domain.ICPressure.RefPatch = "z_upper"
+washita.Geom.domain.ICPressure.FileName = "press.init.pfb"
 
-#-------------------------------------------------------------
+# -------------------------------------------------------------
 # Outputs
 # ------------------------------------------------------------
 
-#Writing output (all pfb):
+# Writing output (all pfb):
 washita.Solver.PrintSubsurfData = False
 washita.Solver.PrintPressure = True
 washita.Solver.PrintSaturation = True
@@ -519,52 +519,52 @@
 washita.Solver.WriteSiloOverlandSum = False
 washita.Solver.WriteSiloCLM = False
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.KnownSolution = 'NoKnownSolution'
+washita.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 # ParFlow Solution
-washita.Solver = 'Richards'
+washita.Solver = "Richards"
 washita.Solver.TerrainFollowingGrid = True
 washita.Solver.Nonlinear.VariableDz = False
 
 washita.Solver.MaxIter = 25000
-washita.Solver.Drop = 1E-20
-washita.Solver.AbsTol = 1E-8
+washita.Solver.Drop = 1e-20
+washita.Solver.AbsTol = 1e-8
 washita.Solver.MaxConvergenceFailures = 8
 washita.Solver.Nonlinear.MaxIter = 80
 washita.Solver.Nonlinear.ResidualTol = 1e-6
 
 ## new solver settings for Terrain Following Grid
-washita.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+washita.Solver.Nonlinear.EtaChoice = "EtaConstant"
 washita.Solver.Nonlinear.EtaValue = 0.001
 washita.Solver.Nonlinear.UseJacobian = True
 washita.Solver.Nonlinear.DerivativeEpsilon = 1e-16
 washita.Solver.Nonlinear.StepTol = 1e-30
-washita.Solver.Nonlinear.Globalization = 'LineSearch'
+washita.Solver.Nonlinear.Globalization = "LineSearch"
 washita.Solver.Linear.KrylovDimension = 70
 washita.Solver.Linear.MaxRestarts = 2
 
-washita.Solver.Linear.Preconditioner = 'PFMGOctree'
-washita.Solver.Linear.Preconditioner.PCMatrixType = 'FullJacobian'
+washita.Solver.Linear.Preconditioner = "PFMGOctree"
+washita.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Distribute CLM and ParFlow inputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.dist('LW.slopex.pfb')
-washita.dist('LW.slopey.pfb')
-washita.dist('IndicatorFile_Gleeson.50z.pfb')
-washita.dist('press.init.pfb')
+washita.dist("LW.slopex.pfb")
+washita.dist("LW.slopey.pfb")
+washita.dist("IndicatorFile_Gleeson.50z.pfb")
+washita.dist("press.init.pfb")
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run Simulation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 washita.run(working_directory=dir_name)
diff --git a/test/python/new_features/clm/washita_clm_keys.py b/test/python/new_features/clm/washita_clm_keys.py
index a9573e377..837ed1481 100644
--- a/test/python/new_features/clm/washita_clm_keys.py
+++ b/test/python/new_features/clm/washita_clm_keys.py
@@ -1,63 +1,63 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # SCRIPT TO RUN LITTLE WASHITA DOMAIN WITH TERRAIN-FOLLOWING GRID
 # DETAILS
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 from parflow import Run
 from parflow.tools.fs import cp, get_absolute_path, rm
 from parflow.tools.builders import CLMImporter
 
 
-washita = Run('washita_clm_keys', __file__)
+washita = Run("washita_clm_keys", __file__)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Import the driver files, and remove them.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 # Set up the computational grid
 washita.ComputationalGrid.NX = 41
 washita.ComputationalGrid.NY = 41
 washita.ComputationalGrid.NZ = 50
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Copy input files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-cp('$PF_SRC/test/tcl/washita/parflow_input/LW.slopex.pfb')
-cp('$PF_SRC/test/tcl/washita/parflow_input/LW.slopey.pfb')
-cp('$PF_SRC/test/tcl/washita/parflow_input/IndicatorFile_Gleeson.50z.pfb')
-cp('$PF_SRC/test/tcl/washita/parflow_input/press.init.pfb')
+cp("$PF_SRC/test/tcl/washita/parflow_input/LW.slopex.pfb")
+cp("$PF_SRC/test/tcl/washita/parflow_input/LW.slopey.pfb")
+cp("$PF_SRC/test/tcl/washita/parflow_input/IndicatorFile_Gleeson.50z.pfb")
+cp("$PF_SRC/test/tcl/washita/parflow_input/press.init.pfb")
 
 
 nldas_files = [
-  'NLDAS.DSWR.000001_to_000024.pfb',
-  'NLDAS.DLWR.000001_to_000024.pfb',
-  'NLDAS.APCP.000001_to_000024.pfb',
-  'NLDAS.Temp.000001_to_000024.pfb',
-  'NLDAS.UGRD.000001_to_000024.pfb',
-  'NLDAS.VGRD.000001_to_000024.pfb',
-  'NLDAS.Press.000001_to_000024.pfb',
-  'NLDAS.SPFH.000001_to_000024.pfb'
-  ]
+    "NLDAS.DSWR.000001_to_000024.pfb",
+    "NLDAS.DLWR.000001_to_000024.pfb",
+    "NLDAS.APCP.000001_to_000024.pfb",
+    "NLDAS.Temp.000001_to_000024.pfb",
+    "NLDAS.UGRD.000001_to_000024.pfb",
+    "NLDAS.VGRD.000001_to_000024.pfb",
+    "NLDAS.Press.000001_to_000024.pfb",
+    "NLDAS.SPFH.000001_to_000024.pfb",
+]
 
 for file in nldas_files:
-  cp('$PF_SRC/test/tcl/washita/NLDAS/' + file)
-  washita.dist(file, R=24)
+    cp("$PF_SRC/test/tcl/washita/NLDAS/" + file)
+    washita.dist(file, R=24)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 washita.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set Processor topology
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 washita.Process.Topology.P = 1
 washita.Process.Topology.Q = 1
 washita.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 washita.ComputationalGrid.Lower.X = 0.0
 washita.ComputationalGrid.Lower.Y = 0.0
@@ -69,22 +69,22 @@
 
 # NX, NY, and NZ already set above
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.GeomInput.Names = 'box_input indi_input'
+washita.GeomInput.Names = "box_input indi_input"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.GeomInput.box_input.InputType = 'Box'
-washita.GeomInput.box_input.GeomName = 'domain'
+washita.GeomInput.box_input.InputType = "Box"
+washita.GeomInput.box_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 washita.Geom.domain.Lower.X = 0.0
 washita.Geom.domain.Lower.Y = 0.0
@@ -93,15 +93,17 @@
 washita.Geom.domain.Upper.X = 41000.0
 washita.Geom.domain.Upper.Y = 41000.0
 washita.Geom.domain.Upper.Z = 100.0
-washita.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+washita.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Indicator Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.GeomInput.indi_input.InputType = 'IndicatorField'
-washita.GeomInput.indi_input.GeomNames = 's1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 g1 g2 g3 g4 g5 g6 g7 g8'
-washita.Geom.indi_input.FileName = 'IndicatorFile_Gleeson.50z.pfb'
+washita.GeomInput.indi_input.InputType = "IndicatorField"
+washita.GeomInput.indi_input.GeomNames = (
+    "s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 g1 g2 g3 g4 g5 g6 g7 g8"
+)
+washita.Geom.indi_input.FileName = "IndicatorFile_Gleeson.50z.pfb"
 
 washita.GeomInput.s1.Value = 1
 washita.GeomInput.s2.Value = 2
@@ -125,226 +127,226 @@
 washita.GeomInput.g7.Value = 27
 washita.GeomInput.g8.Value = 28
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Permeability (values in m/hr)
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Geom.Perm.Names = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9 g2 g3 g6 g8'
+washita.Geom.Perm.Names = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9 g2 g3 g6 g8"
 
-washita.Geom.domain.Perm.Type = 'Constant'
+washita.Geom.domain.Perm.Type = "Constant"
 washita.Geom.domain.Perm.Value = 0.2
 
-washita.Geom.s1.Perm.Type = 'Constant'
+washita.Geom.s1.Perm.Type = "Constant"
 washita.Geom.s1.Perm.Value = 0.269022595
 
-washita.Geom.s2.Perm.Type = 'Constant'
+washita.Geom.s2.Perm.Type = "Constant"
 washita.Geom.s2.Perm.Value = 0.043630356
 
-washita.Geom.s3.Perm.Type = 'Constant'
+washita.Geom.s3.Perm.Type = "Constant"
 washita.Geom.s3.Perm.Value = 0.015841225
 
-washita.Geom.s4.Perm.Type = 'Constant'
+washita.Geom.s4.Perm.Type = "Constant"
 washita.Geom.s4.Perm.Value = 0.007582087
 
-washita.Geom.s5.Perm.Type = 'Constant'
+washita.Geom.s5.Perm.Type = "Constant"
 washita.Geom.s5.Perm.Value = 0.01818816
 
-washita.Geom.s6.Perm.Type = 'Constant'
+washita.Geom.s6.Perm.Type = "Constant"
 washita.Geom.s6.Perm.Value = 0.005009435
 
-washita.Geom.s7.Perm.Type = 'Constant'
+washita.Geom.s7.Perm.Type = "Constant"
 washita.Geom.s7.Perm.Value = 0.005492736
 
-washita.Geom.s8.Perm.Type = 'Constant'
+washita.Geom.s8.Perm.Type = "Constant"
 washita.Geom.s8.Perm.Value = 0.004675077
 
-washita.Geom.s9.Perm.Type = 'Constant'
+washita.Geom.s9.Perm.Type = "Constant"
 washita.Geom.s9.Perm.Value = 0.003386794
 
-washita.Geom.g2.Perm.Type = 'Constant'
+washita.Geom.g2.Perm.Type = "Constant"
 washita.Geom.g2.Perm.Value = 0.025
 
-washita.Geom.g3.Perm.Type = 'Constant'
+washita.Geom.g3.Perm.Type = "Constant"
 washita.Geom.g3.Perm.Value = 0.059
 
-washita.Geom.g6.Perm.Type = 'Constant'
+washita.Geom.g6.Perm.Type = "Constant"
 washita.Geom.g6.Perm.Value = 0.2
 
-washita.Geom.g8.Perm.Type = 'Constant'
+washita.Geom.g8.Perm.Type = "Constant"
 washita.Geom.g8.Perm.Value = 0.68
 
-washita.Perm.TensorType = 'TensorByGeom'
-washita.Geom.Perm.TensorByGeom.Names = 'domain'
+washita.Perm.TensorType = "TensorByGeom"
+washita.Geom.Perm.TensorByGeom.Names = "domain"
 washita.Geom.domain.Perm.TensorValX = 1.0
 washita.Geom.domain.Perm.TensorValY = 1.0
 washita.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.SpecificStorage.Type = 'Constant'
-washita.SpecificStorage.GeomNames = 'domain'
+washita.SpecificStorage.Type = "Constant"
+washita.SpecificStorage.GeomNames = "domain"
 washita.Geom.domain.SpecificStorage.Value = 1.0e-5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Phase.Names = 'water'
-washita.Phase.water.Density.Type = 'Constant'
+washita.Phase.Names = "water"
+washita.Phase.water.Density.Type = "Constant"
 washita.Phase.water.Density.Value = 1.0
-washita.Phase.water.Viscosity.Type = 'Constant'
+washita.Phase.water.Viscosity.Type = "Constant"
 washita.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Contaminants.Names = ''
+washita.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 washita.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Timing (time units is set by units of permeability)
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 washita.TimingInfo.BaseUnit = 1.0
 washita.TimingInfo.StartCount = 0
 washita.TimingInfo.StartTime = 0.0
 washita.TimingInfo.StopTime = 12.0
 washita.TimingInfo.DumpInterval = 24.0
-washita.TimeStep.Type = 'Constant'
+washita.TimeStep.Type = "Constant"
 washita.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Geom.Porosity.GeomNames = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9'
+washita.Geom.Porosity.GeomNames = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9"
 
-washita.Geom.domain.Porosity.Type = 'Constant'
+washita.Geom.domain.Porosity.Type = "Constant"
 washita.Geom.domain.Porosity.Value = 0.4
 
-washita.Geom.s1.Porosity.Type = 'Constant'
+washita.Geom.s1.Porosity.Type = "Constant"
 washita.Geom.s1.Porosity.Value = 0.375
 
-washita.Geom.s2.Porosity.Type = 'Constant'
+washita.Geom.s2.Porosity.Type = "Constant"
 washita.Geom.s2.Porosity.Value = 0.39
 
-washita.Geom.s3.Porosity.Type = 'Constant'
+washita.Geom.s3.Porosity.Type = "Constant"
 washita.Geom.s3.Porosity.Value = 0.387
 
-washita.Geom.s4.Porosity.Type = 'Constant'
+washita.Geom.s4.Porosity.Type = "Constant"
 washita.Geom.s4.Porosity.Value = 0.439
 
-washita.Geom.s5.Porosity.Type = 'Constant'
+washita.Geom.s5.Porosity.Type = "Constant"
 washita.Geom.s5.Porosity.Value = 0.489
 
-washita.Geom.s6.Porosity.Type = 'Constant'
+washita.Geom.s6.Porosity.Type = "Constant"
 washita.Geom.s6.Porosity.Value = 0.399
 
-washita.Geom.s7.Porosity.Type = 'Constant'
+washita.Geom.s7.Porosity.Type = "Constant"
 washita.Geom.s7.Porosity.Value = 0.384
 
-washita.Geom.s8.Porosity.Type = 'Constant'
+washita.Geom.s8.Porosity.Type = "Constant"
 washita.Geom.s8.Porosity.Value = 0.482
 
-washita.Geom.s9.Porosity.Type = 'Constant'
+washita.Geom.s9.Porosity.Type = "Constant"
 washita.Geom.s9.Porosity.Value = 0.442
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Domain.GeomName = 'domain'
+washita.Domain.GeomName = "domain"
 
-#----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 # Mobility
-#----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 
-washita.Phase.water.Mobility.Type = 'Constant'
+washita.Phase.water.Mobility.Type = "Constant"
 washita.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Wells.Names = ''
+washita.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Cycle.Names = 'constant'
-washita.Cycle.constant.Names = 'alltime'
+washita.Cycle.Names = "constant"
+washita.Cycle.constant.Names = "alltime"
 washita.Cycle.constant.alltime.Length = 1
 washita.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 washita.BCPressure.PatchNames = washita.Geom.domain.Patches
 
-washita.Patch.x_lower.BCPressure.Type = 'FluxConst'
-washita.Patch.x_lower.BCPressure.Cycle = 'constant'
+washita.Patch.x_lower.BCPressure.Type = "FluxConst"
+washita.Patch.x_lower.BCPressure.Cycle = "constant"
 washita.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-washita.Patch.y_lower.BCPressure.Type = 'FluxConst'
-washita.Patch.y_lower.BCPressure.Cycle = 'constant'
+washita.Patch.y_lower.BCPressure.Type = "FluxConst"
+washita.Patch.y_lower.BCPressure.Cycle = "constant"
 washita.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-washita.Patch.z_lower.BCPressure.Type = 'FluxConst'
-washita.Patch.z_lower.BCPressure.Cycle = 'constant'
+washita.Patch.z_lower.BCPressure.Type = "FluxConst"
+washita.Patch.z_lower.BCPressure.Cycle = "constant"
 washita.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-washita.Patch.x_upper.BCPressure.Type = 'FluxConst'
-washita.Patch.x_upper.BCPressure.Cycle = 'constant'
+washita.Patch.x_upper.BCPressure.Type = "FluxConst"
+washita.Patch.x_upper.BCPressure.Cycle = "constant"
 washita.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-washita.Patch.y_upper.BCPressure.Type = 'FluxConst'
-washita.Patch.y_upper.BCPressure.Cycle = 'constant'
+washita.Patch.y_upper.BCPressure.Type = "FluxConst"
+washita.Patch.y_upper.BCPressure.Cycle = "constant"
 washita.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-washita.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-washita.Patch.z_upper.BCPressure.Cycle = 'constant'
+washita.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+washita.Patch.z_upper.BCPressure.Cycle = "constant"
 washita.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Topo slopes in x-direction
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.TopoSlopesX.Type = 'PFBFile'
-washita.TopoSlopesX.GeomNames = 'domain'
-washita.TopoSlopesX.FileName = 'LW.slopex.pfb'
+washita.TopoSlopesX.Type = "PFBFile"
+washita.TopoSlopesX.GeomNames = "domain"
+washita.TopoSlopesX.FileName = "LW.slopex.pfb"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Topo slopes in y-direction
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.TopoSlopesY.Type = 'PFBFile'
-washita.TopoSlopesY.GeomNames = 'domain'
-washita.TopoSlopesY.FileName = 'LW.slopey.pfb'
+washita.TopoSlopesY.Type = "PFBFile"
+washita.TopoSlopesY.GeomNames = "domain"
+washita.TopoSlopesY.FileName = "LW.slopey.pfb"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mannings coefficient
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Mannings.Type = 'Constant'
-washita.Mannings.GeomNames = 'domain'
+washita.Mannings.Type = "Constant"
+washita.Mannings.GeomNames = "domain"
 washita.Mannings.Geom.domain.Value = 5.52e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Phase.RelPerm.Type = 'VanGenuchten'
-washita.Phase.RelPerm.GeomNames = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9 '
+washita.Phase.RelPerm.Type = "VanGenuchten"
+washita.Phase.RelPerm.GeomNames = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9 "
 
 washita.Geom.domain.RelPerm.Alpha = 3.5
 washita.Geom.domain.RelPerm.N = 2.0
@@ -376,15 +378,15 @@
 washita.Geom.s9.RelPerm.Alpha = 1.585
 washita.Geom.s9.RelPerm.N = 2.413
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Saturation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.Phase.Saturation.Type = 'VanGenuchten'
-washita.Phase.Saturation.GeomNames = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9 '
+washita.Phase.Saturation.Type = "VanGenuchten"
+washita.Phase.Saturation.GeomNames = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9 "
 
 washita.Geom.domain.Saturation.Alpha = 3.5
-washita.Geom.domain.Saturation.N = 2.
+washita.Geom.domain.Saturation.N = 2.0
 washita.Geom.domain.Saturation.SRes = 0.2
 washita.Geom.domain.Saturation.SSat = 1.0
 
@@ -433,55 +435,55 @@
 washita.Geom.s9.Saturation.SRes = 0.000001
 washita.Geom.s9.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.PhaseSources.water.Type = 'Constant'
-washita.PhaseSources.water.GeomNames = 'domain'
+washita.PhaseSources.water.Type = "Constant"
+washita.PhaseSources.water.GeomNames = "domain"
 washita.PhaseSources.water.Geom.domain.Value = 0.0
 
-#----------------------------------------------------------------
+# ----------------------------------------------------------------
 # CLM Settings:
 # ------------------------------------------------------------
 
-washita.Solver.LSM = 'CLM'
-washita.Solver.CLM.CLMFileDir = '.'
+washita.Solver.LSM = "CLM"
+washita.Solver.CLM.CLMFileDir = "."
 washita.Solver.CLM.Print1dOut = False
 washita.Solver.CLM.CLMDumpInterval = 1
 
-washita.Solver.CLM.MetForcing = '3D'
-washita.Solver.CLM.MetFileName = 'NLDAS'
-washita.Solver.CLM.MetFilePath = '.'
+washita.Solver.CLM.MetForcing = "3D"
+washita.Solver.CLM.MetFileName = "NLDAS"
+washita.Solver.CLM.MetFilePath = "."
 washita.Solver.CLM.MetFileNT = 24
 washita.Solver.CLM.IstepStart = 1
 
-washita.Solver.CLM.EvapBeta = 'Linear'
-washita.Solver.CLM.VegWaterStress = 'Saturation'
+washita.Solver.CLM.EvapBeta = "Linear"
+washita.Solver.CLM.VegWaterStress = "Saturation"
 washita.Solver.CLM.ResSat = 0.1
 washita.Solver.CLM.WiltingPoint = 0.12
 washita.Solver.CLM.FieldCapacity = 0.98
-washita.Solver.CLM.IrrigationType = 'none'
+washita.Solver.CLM.IrrigationType = "none"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # New keys to generate drv files for CLM
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-washita.Solver.CLM.Input.File.ActiveRestart = 'clm.rst.'
-washita.Solver.CLM.Input.File.MetInput = 'narr_1hr.dat'
-washita.Solver.CLM.Input.File.Output = 'clm_out.txt'
-washita.Solver.CLM.Input.File.ParamOutput = 'clm_para_out.dat'
-washita.Solver.CLM.Input.File.VegTileSpecification = 'drv_vegm.alluv.dat'
+washita.Solver.CLM.Input.File.ActiveRestart = "clm.rst."
+washita.Solver.CLM.Input.File.MetInput = "narr_1hr.dat"
+washita.Solver.CLM.Input.File.Output = "clm_out.txt"
+washita.Solver.CLM.Input.File.ParamOutput = "clm_para_out.dat"
+washita.Solver.CLM.Input.File.VegTileSpecification = "drv_vegm.alluv.dat"
 
 washita.Solver.CLM.Input.ICSource.Code = 2
 washita.Solver.CLM.Input.InitCond.SnowCover = 0
 
 # Time range
 washita.Solver.CLM.Input.Timing.RestartCode = 2
-washita.Solver.CLM.Input.Timing.StartDate = '1998/10/01'
-washita.Solver.CLM.Input.Timing.StartTime = '05:00:00'
-washita.Solver.CLM.Input.Timing.EndDate = '1999/10/01'
-washita.Solver.CLM.Input.Timing.EndTime = '04:00:00'
+washita.Solver.CLM.Input.Timing.StartDate = "1998/10/01"
+washita.Solver.CLM.Input.Timing.StartTime = "05:00:00"
+washita.Solver.CLM.Input.Timing.EndDate = "1999/10/01"
+washita.Solver.CLM.Input.Timing.EndTime = "04:00:00"
 
 # Default constants
 washita.Solver.CLM.Vegetation.Map.Latitude.Value = 34.75
@@ -491,28 +493,32 @@
 washita.Solver.CLM.Vegetation.Map.Sand.Value = 0.16
 
 # Setup default land cover names with corresponding vegetation parameters
-washita.Solver.CLM.Vegetation.Parameters.pfset(yaml_file='clm_input/washita_veg_params_ref.yaml')
+washita.Solver.CLM.Vegetation.Parameters.pfset(
+    yaml_file="clm_input/washita_veg_params_ref.yaml"
+)
 
 # Override only existing landfrac pfb files
 land_frac = washita.Solver.CLM.Vegetation.Map.LandFrac
-for land_cover_name in ['croplands', 'forest_dn', 'grasslands', 'savannas']:
-  land_frac[land_cover_name].Type = 'PFBFile'
-  land_frac[land_cover_name].FileName = f'clm_input/washita_clm_{land_cover_name}_landfrac.pfb'
+for land_cover_name in ["croplands", "forest_dn", "grasslands", "savannas"]:
+    land_frac[land_cover_name].Type = "PFBFile"
+    land_frac[land_cover_name].FileName = (
+        f"clm_input/washita_clm_{land_cover_name}_landfrac.pfb"
+    )
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-washita.ICPressure.Type = 'PFBFile'
-washita.ICPressure.GeomNames = 'domain'
-washita.Geom.domain.ICPressure.RefPatch = 'z_upper'
-washita.Geom.domain.ICPressure.FileName = 'press.init.pfb'
+washita.ICPressure.Type = "PFBFile"
+washita.ICPressure.GeomNames = "domain"
+washita.Geom.domain.ICPressure.RefPatch = "z_upper"
+washita.Geom.domain.ICPressure.FileName = "press.init.pfb"
 
-#-------------------------------------------------------------
+# -------------------------------------------------------------
 # Outputs
 # ------------------------------------------------------------
 
-#Writing output (all pfb):
+# Writing output (all pfb):
 washita.Solver.PrintSubsurfData = False
 washita.Solver.PrintPressure = True
 washita.Solver.PrintSaturation = True
@@ -532,55 +538,55 @@
 washita.Solver.WriteSiloOverlandSum = False
 washita.Solver.WriteSiloCLM = False
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.KnownSolution = 'NoKnownSolution'
+washita.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 # ParFlow Solution
-washita.Solver = 'Richards'
+washita.Solver = "Richards"
 washita.Solver.TerrainFollowingGrid = True
 washita.Solver.Nonlinear.VariableDz = False
 
 washita.Solver.MaxIter = 25000
-washita.Solver.Drop = 1E-20
-washita.Solver.AbsTol = 1E-8
+washita.Solver.Drop = 1e-20
+washita.Solver.AbsTol = 1e-8
 washita.Solver.MaxConvergenceFailures = 8
 washita.Solver.Nonlinear.MaxIter = 80
 washita.Solver.Nonlinear.ResidualTol = 1e-6
 
 ## new solver settings for Terrain Following Grid
-washita.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+washita.Solver.Nonlinear.EtaChoice = "EtaConstant"
 washita.Solver.Nonlinear.EtaValue = 0.001
 washita.Solver.Nonlinear.UseJacobian = True
 washita.Solver.Nonlinear.DerivativeEpsilon = 1e-16
 washita.Solver.Nonlinear.StepTol = 1e-30
-washita.Solver.Nonlinear.Globalization = 'LineSearch'
+washita.Solver.Nonlinear.Globalization = "LineSearch"
 washita.Solver.Linear.KrylovDimension = 70
 washita.Solver.Linear.MaxRestarts = 2
 
-washita.Solver.Linear.Preconditioner = 'PFMGOctree'
-washita.Solver.Linear.Preconditioner.PCMatrixType = 'FullJacobian'
+washita.Solver.Linear.Preconditioner = "PFMGOctree"
+washita.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Distribute CLM and ParFlow inputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-washita.dist('LW.slopex.pfb')
-washita.dist('LW.slopey.pfb')
-washita.dist('IndicatorFile_Gleeson.50z.pfb')
-washita.dist('press.init.pfb')
+washita.dist("LW.slopex.pfb")
+washita.dist("LW.slopey.pfb")
+washita.dist("IndicatorFile_Gleeson.50z.pfb")
+washita.dist("press.init.pfb")
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run Simulation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 # Write out the yaml file
-washita.write('washita_clm_keys', file_format='yaml')
+washita.write("washita_clm_keys", file_format="yaml")
 
 washita.run()
diff --git a/test/python/new_features/clm_build_export.py b/test/python/new_features/clm_build_export.py
index caf2510ef..744715468 100644
--- a/test/python/new_features/clm_build_export.py
+++ b/test/python/new_features/clm_build_export.py
@@ -8,43 +8,45 @@
 from parflow.tools.fs import get_absolute_path
 from parflow.tools.io import read_pfb, read_clm
 
-clmin_file = '../../input/drv_clmin.dat.old'
-vegm_file = '../../tcl/clm/drv_vegm.dat'
-vegp_file = '../../tcl/clm/drv_vegp.dat'
+clmin_file = "../../input/drv_clmin.dat.old"
+vegm_file = "../../tcl/clm/drv_vegm.dat"
+vegp_file = "../../tcl/clm/drv_vegp.dat"
 
 
 # ---------------------------------------------------------
 # Testing clm data key setting with DomainBuilder
 # ---------------------------------------------------------
 
+
 def test_clmin_key_setting(clm):
     clm_key_settings = {
-        'StartDate': '2020-01-01',
-        'StartTime': '00-00-00',
-        'EndDate': '2020-12-31',
-        'EndTime': '23-59-59',
-        'metf1d': 'narr_1hr.sc3.txt',
-        'outf1d': 'washita.output.txt',
-        'poutf1d': 'test.out.dat',
-        'rstf': 'washita.rst.'
+        "StartDate": "2020-01-01",
+        "StartTime": "00-00-00",
+        "EndDate": "2020-12-31",
+        "EndTime": "23-59-59",
+        "metf1d": "narr_1hr.sc3.txt",
+        "outf1d": "washita.output.txt",
+        "poutf1d": "test.out.dat",
+        "rstf": "washita.rst.",
     }
     DomainBuilder(clm).clm_input(**clm_key_settings)
 
     clm_timing = clm.Solver.CLM.Input.Timing
     clm_files = clm.Solver.CLM.Input.File
-    assert clm_timing.StartYear == '2020'
-    assert clm_timing.EndHour == '23'
-    assert clm_timing.EndMinute == '59'
-    assert clm_files.MetInput == clm_key_settings['metf1d']
-    assert clm_files.Output == clm_key_settings['outf1d']
-    assert clm_files.ParamOutput == clm_key_settings['poutf1d']
-    assert clm_files.ActiveRestart == clm_key_settings['rstf']
+    assert clm_timing.StartYear == "2020"
+    assert clm_timing.EndHour == "23"
+    assert clm_timing.EndMinute == "59"
+    assert clm_files.MetInput == clm_key_settings["metf1d"]
+    assert clm_files.Output == clm_key_settings["outf1d"]
+    assert clm_files.ParamOutput == clm_key_settings["poutf1d"]
+    assert clm_files.ActiveRestart == clm_key_settings["rstf"]
 
 
 # ---------------------------------------------------------
 # Testing clmin file setting with CLMImporter
 # ---------------------------------------------------------
 
+
 def test_clmin_file_setting(clm):
     input = clm.Solver.CLM.Input
 
@@ -62,6 +64,7 @@ def test_clmin_file_setting(clm):
 # Testing vegm file setting with CLMImporter
 # ---------------------------------------------------------
 
+
 def test_vegm_file_setting(clm):
     veg_params = clm.Solver.CLM.Vegetation.Parameters
 
@@ -70,7 +73,7 @@ def test_vegm_file_setting(clm):
         # land items yet.
         CLMImporter(clm).map_file(vegm_file)
     except Exception as e:
-        assert str(e) == 'Land cover items are not set'
+        assert str(e) == "Land cover items are not set"
 
     # Now set the land cover items and try again
     veg_params.LandNames = veg_params.LandNames
@@ -83,6 +86,7 @@ def test_vegm_file_setting(clm):
 # Testing vegp file setting with CLMImporter
 # ---------------------------------------------------------
 
+
 def test_vegp_file_setting(clm):
     CLMImporter(clm).parameters_file(vegp_file)
     verify_vegp_data(clm)
@@ -92,26 +96,27 @@ def test_vegp_file_setting(clm):
 # Testing exporters
 # ---------------------------------------------------------
 
+
 def test_clmin_round_trip(clm):
-    path = Path(get_absolute_path('drv_clmin.dat'))
+    path = Path(get_absolute_path("drv_clmin.dat"))
     if path.exists():
         path.unlink()
 
     CLMExporter(clm).write_input()
 
-    new_clm = Run('clm', __file__)
+    new_clm = Run("clm", __file__)
     CLMImporter(new_clm).input_file(path)
     verify_clmin_data(new_clm)
 
 
 def test_vegm_round_trip(clm):
-    path = Path(get_absolute_path('drv_vegm.dat'))
+    path = Path(get_absolute_path("drv_vegm.dat"))
     if path.exists():
         path.unlink()
 
     CLMExporter(clm).write_map()
 
-    new_clm = Run('clm', __file__)
+    new_clm = Run("clm", __file__)
     veg_params = new_clm.Solver.CLM.Vegetation.Parameters
 
     # Set the land cover items
@@ -122,13 +127,13 @@ def test_vegm_round_trip(clm):
 
 
 def test_vegp_round_trip(clm):
-    path = Path(get_absolute_path('drv_vegp.dat'))
+    path = Path(get_absolute_path("drv_vegp.dat"))
     if path.exists():
         path.unlink()
 
     CLMExporter(clm).write_parameters()
 
-    new_clm = Run('clm', __file__)
+    new_clm = Run("clm", __file__)
     veg_params = new_clm.Solver.CLM.Vegetation.Parameters
 
     # Set the land cover items
@@ -139,41 +144,39 @@ def test_vegp_round_trip(clm):
 
 
 def test_fewer_land_names(clm):
-    vegm_file = Path(get_absolute_path('drv_vegm.dat'))
-    vegp_file = Path(get_absolute_path('drv_vegp.dat'))
+    vegm_file = Path(get_absolute_path("drv_vegm.dat"))
+    vegp_file = Path(get_absolute_path("drv_vegp.dat"))
     for path in (vegm_file, vegp_file):
         if path.exists():
             path.unlink()
 
     CLMExporter(clm).write_map().write_parameters()
 
-    new_clm = Run('clm', __file__)
+    new_clm = Run("clm", __file__)
     veg_params = new_clm.Solver.CLM.Vegetation.Parameters
     veg_map = new_clm.Solver.CLM.Vegetation.Map
 
     # Set only two land cover items
-    veg_params.LandNames = 'forest_en forest_eb'
+    veg_params.LandNames = "forest_en forest_eb"
 
-    CLMImporter(new_clm) \
-        .map_file(vegm_file) \
-        .parameters_file(vegp_file)
+    CLMImporter(new_clm).map_file(vegm_file).parameters_file(vegp_file)
 
-    assert veg_params.LandNames == ['forest_en', 'forest_eb']
-    assert 'water' not in veg_params.keys()
-    assert 'water' not in veg_map.keys()
+    assert veg_params.LandNames == ["forest_en", "forest_eb"]
+    assert "water" not in veg_params.keys()
+    assert "water" not in veg_map.keys()
 
     veg_map.LandFrac.forest_en.Value = 0.6
     veg_map.LandFrac.forest_eb.Value = 0.4
-    veg_map.LandFrac.forest_en.Type = 'Constant'
-    veg_map.LandFrac.forest_eb.Type = 'Constant'
+    veg_map.LandFrac.forest_en.Type = "Constant"
+    veg_map.LandFrac.forest_eb.Type = "Constant"
 
     new_clm.ComputationalGrid.NX = 5
     new_clm.ComputationalGrid.NY = 5
 
     CLMExporter(new_clm).write_map().write_parameters()
 
-    vegm_data = read_clm(vegm_file, type='vegm')
-    vegp_data = read_clm(vegp_file, type='vegp')
+    vegm_data = read_clm(vegm_file, type="vegm")
+    vegp_data = read_clm(vegp_file, type="vegp")
 
     # These should be 0.6 and 0.4
     shape = vegm_data.shape[:2]
@@ -194,7 +197,7 @@ def test_fewer_land_names(clm):
             assert val[i] == default_vegp[key]
 
     # Make sure at least one of the other values is correct
-    key = 'z0m'
+    key = "z0m"
     ind = 1
     val = 2.2
     assert vegp_data[key][ind] != default_vegp[key]
@@ -218,43 +221,43 @@ def verify_vegm_data(clm):
     veg_map = clm.Solver.CLM.Vegetation.Map
 
     shape = (5, 5)
-    properties = ['Latitude', 'Longitude', 'Sand', 'Clay', 'Color', 'LandFrac']
+    properties = ["Latitude", "Longitude", "Sand", "Clay", "Color", "LandFrac"]
 
     # Check to see that all matrices got set
     for key in veg_map.keys():
         item = veg_map[key]
 
-        if key == 'LandFrac':
+        if key == "LandFrac":
             for land_frac_name in item.keys():
                 land_frac = item[land_frac_name]
-                assert land_frac.Type in ('PFBFile', 'Constant')
-                if land_frac.Type == 'PFBFile':
+                assert land_frac.Type in ("PFBFile", "Constant")
+                if land_frac.Type == "PFBFile":
                     array = read_pfb(land_frac.FileName).squeeze()
                     assert array.shape == shape
         else:
-            assert item.Type in ('PFBFile', 'Constant')
-            if item.Type == 'PFBFile':
+            assert item.Type in ("PFBFile", "Constant")
+            if item.Type == "PFBFile":
                 array = read_pfb(item.FileName).squeeze()
                 assert array.shape == shape
 
     # Check a few individual ones
     const_values = {
-        'Latitude': 34.75,
-        'Longitude': -98.138,
-        'Sand': 0.16,
-        'Clay': 0.265,
-        'Color': 2
+        "Latitude": 34.75,
+        "Longitude": -98.138,
+        "Sand": 0.16,
+        "Clay": 0.265,
+        "Color": 2,
     }
     for name, val in const_values.items():
-        assert veg_map[name].Type == 'Constant'
+        assert veg_map[name].Type == "Constant"
         assert veg_map[name].Value == val
 
     const_land_fracs = {
-        'forest_en': 0.0,
-        'grasslands': 1.0,
+        "forest_en": 0.0,
+        "grasslands": 1.0,
     }
     for name, val in const_land_fracs.items():
-        assert veg_map.LandFrac[name].Type == 'Constant'
+        assert veg_map.LandFrac[name].Type == "Constant"
         assert veg_map.LandFrac[name].Value == val
 
 
@@ -263,26 +266,21 @@ def verify_vegp_data(clm):
 
     # Check that a few of these were set
     check_list = {
-        'forest_en': {
-            'WaterType': 1,
-            'MaxLAI': 6.0,
-            'MinLAI': 5.0,
-            'DispHeight': 11.0
+        "forest_en": {"WaterType": 1, "MaxLAI": 6.0, "MinLAI": 5.0, "DispHeight": 11.0},
+        "water": {
+            "WaterType": 3,
+            "StemAI": 2.0,
+            "RoughLength": 0.002,
+            "StemTransNir": -99.0,
         },
-        'water': {
-            'WaterType': 3,
-            'StemAI': 2.0,
-            'RoughLength': 0.002,
-            'StemTransNir': -99.0
-        }
     }
     for name, item in check_list.items():
         for key, val in item.items():
             assert veg_params[name][key] == val
 
 
-if __name__ == '__main__':
-    clm = Run('clm', __file__)
+if __name__ == "__main__":
+    clm = Run("clm", __file__)
 
     clm.ComputationalGrid.NX = 5
     clm.ComputationalGrid.NY = 5
diff --git a/test/python/new_features/clm_file_handling.py b/test/python/new_features/clm_file_handling.py
index 251d409d3..1a0887c1d 100644
--- a/test/python/new_features/clm_file_handling.py
+++ b/test/python/new_features/clm_file_handling.py
@@ -13,8 +13,8 @@
 def verify_clmin_data(data):
     assert data
 
-    path = get_absolute_path('$PF_SRC/test/correct_output/clmin_data.ref.json')
-    with open(path, 'r') as rf:
+    path = get_absolute_path("$PF_SRC/test/correct_output/clmin_data.ref.json")
+    with open(path, "r") as rf:
         ref_data = json.load(rf)
 
     assert data == ref_data
@@ -28,7 +28,7 @@ def verify_vegm_data(data):
     assert data[1, 2, 3] == 0.265
 
     # Now test the whole array
-    path = get_absolute_path('$PF_SRC/test/correct_output/drv_vegm.ref.npy')
+    path = get_absolute_path("$PF_SRC/test/correct_output/drv_vegm.ref.npy")
     ref_data = np.load(path)
     assert np.array_equal(data, ref_data)
 
@@ -36,8 +36,8 @@ def verify_vegm_data(data):
 def verify_vegp_data(data):
     assert data
 
-    path = get_absolute_path('$PF_SRC/test/correct_output/vegp_data.ref.json')
-    with open(path, 'r') as rf:
+    path = get_absolute_path("$PF_SRC/test/correct_output/vegp_data.ref.json")
+    with open(path, "r") as rf:
         ref_data = json.load(rf)
 
     assert data == ref_data
@@ -55,25 +55,25 @@ def verify_short_vegp_data(exporter, data):
 
 clm = Run("clm", __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Testing clm data readers
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # Reading drv_clmin.dat into dictionary
 # using old file that has more variables than CLM currently needs
-clmin_data = read_clm('$PF_SRC/test/input/drv_clmin.dat.old', type='clmin')
+clmin_data = read_clm("$PF_SRC/test/input/drv_clmin.dat.old", type="clmin")
 verify_clmin_data(clmin_data)
-print(f'clmin_data = {clmin_data}')
+print(f"clmin_data = {clmin_data}")
 
 # Reading drv_vegm.dat into 3D array
-vegm_data = read_clm('$PF_SRC/test/tcl/clm/drv_vegm.dat', type='vegm')
+vegm_data = read_clm("$PF_SRC/test/tcl/clm/drv_vegm.dat", type="vegm")
 verify_vegm_data(vegm_data)
-print(f'vegm_data.shape = {vegm_data.shape}')
+print(f"vegm_data.shape = {vegm_data.shape}")
 
 # Reading drv_vegp.dat into dictionary
-vegp_data = read_clm('$PF_SRC/test/tcl/clm/drv_vegp.dat', type='vegp')
+vegp_data = read_clm("$PF_SRC/test/tcl/clm/drv_vegp.dat", type="vegp")
 verify_vegp_data(vegp_data)
-print(f'vegp_data = {vegp_data}')
+print(f"vegp_data = {vegp_data}")
 
 # ---------------------------------------------------------
 # Testing clm data writers
@@ -81,24 +81,21 @@ def verify_short_vegp_data(exporter, data):
 
 # Remove any leftover driver files...
 paths = [
-    'drv_clmin.dat',
-    'drv_vegm.dat',
-    'drv_vegp.dat',
+    "drv_clmin.dat",
+    "drv_vegm.dat",
+    "drv_vegp.dat",
 ]
 for path in paths:
-    path = Path(get_absolute_path(f'$PF_SRC/test/python/new_features/{path}'))
+    path = Path(get_absolute_path(f"$PF_SRC/test/python/new_features/{path}"))
     if path.exists():
         path.unlink()
 
-CLMExporter(clm) \
-    .write_input() \
-    .write_map(vegm_data) \
-    .write_parameters(vegp_data)
+CLMExporter(clm).write_input().write_map(vegm_data).write_parameters(vegp_data)
 
 # Re-read the exported files back in to ensure they are the same
-new_clmin_data = read_clm('drv_clmin.dat', type='clmin')
-new_vegm_data = read_clm('drv_vegm.dat', type='vegm')
-new_vegp_data = read_clm('drv_vegp.dat', type='vegp')
+new_clmin_data = read_clm("drv_clmin.dat", type="clmin")
+new_vegm_data = read_clm("drv_vegm.dat", type="vegm")
+new_vegp_data = read_clm("drv_vegp.dat", type="vegp")
 
 # FIXME: because some CLM keys are not yet registered in the pf-keys yaml
 # files, the new clm data does not match.
@@ -106,7 +103,9 @@ def verify_short_vegp_data(exporter, data):
 assert np.array_equal(new_vegm_data, vegm_data)
 assert new_vegp_data == vegp_data
 
-clm.Solver.CLM.Vegetation.Parameters.LandNames = 'forest_en forest_eb forest_dn forest_db'
+clm.Solver.CLM.Vegetation.Parameters.LandNames = (
+    "forest_en forest_eb forest_dn forest_db"
+)
 
 # Make the data list short and ensure default values get filled
 for val in vegp_data.values():
@@ -114,12 +113,10 @@ def verify_short_vegp_data(exporter, data):
         val.pop()
 
 CLMExporter(clm).write_parameters(vegp_data)
-new_vegp_data = read_clm('drv_vegp.dat', type='vegp')
+new_vegp_data = read_clm("drv_vegp.dat", type="vegp")
 verify_short_vegp_data(CLMExporter(clm), new_vegp_data)
 
-VegParamBuilder(clm) \
-    .load_default_properties() \
-    .print()
+VegParamBuilder(clm).load_default_properties().print()
 
-clm.Solver.CLM.Vegetation.Map.Clay.Type = 'Constant'
+clm.Solver.CLM.Vegetation.Map.Clay.Type = "Constant"
 clm.Solver.CLM.Vegetation.Map.Clay.Value = 0.264
diff --git a/test/python/new_features/clm_overwrite.py b/test/python/new_features/clm_overwrite.py
index 6583077e6..66ed2300f 100644
--- a/test/python/new_features/clm_overwrite.py
+++ b/test/python/new_features/clm_overwrite.py
@@ -9,15 +9,15 @@
 
 
 default_driver_file_names = {
-    'input': 'drv_clmin.dat',
-    'map': 'drv_vegm.dat',
-    'parameters': 'drv_vegp.dat',
+    "input": "drv_clmin.dat",
+    "map": "drv_vegm.dat",
+    "parameters": "drv_vegp.dat",
 }
 
 default_driver_file_paths = {
-    'input': '../../input/drv_clmin.dat.old',
-    'map': '../../tcl/clm/drv_vegm.dat',
-    'parameters': '../../tcl/clm/drv_vegp.dat',
+    "input": "../../input/drv_clmin.dat.old",
+    "map": "../../tcl/clm/drv_vegm.dat",
+    "parameters": "../../tcl/clm/drv_vegp.dat",
 }
 
 
@@ -25,14 +25,13 @@
 # Testing overwriting driver files with CLMExporter
 # ---------------------------------------------------------
 
+
 def test_overwrite_failure(clm):
     # First, copy the files over
     copy_driver_files()
 
     # Import the files
-    CLMImporter(clm) \
-        .set_default_land_names() \
-        .files()
+    CLMImporter(clm).set_default_land_names().files()
 
     exporter = CLMExporter(clm)
 
@@ -55,9 +54,7 @@ def test_overwrite_success(clm):
     copy_driver_files()
 
     # Import the files
-    CLMImporter(clm) \
-        .set_default_land_names() \
-        .files()
+    CLMImporter(clm).set_default_land_names().files()
 
     exporter = CLMExporter(clm)
 
@@ -125,13 +122,11 @@ def files_unchanged():
         return originally_written_times == driver_file_written_times()
 
     # Import the files
-    CLMImporter(clm) \
-        .set_default_land_names() \
-        .files()
+    CLMImporter(clm).set_default_land_names().files()
 
     exporter = CLMExporter(clm)
 
-    warning_func = '_print_not_written_warning'
+    warning_func = "_print_not_written_warning"
     # This should not print any warnings even though it didn't write,
     # because we did not change any settings.
     assert was_called(exporter, warning_func, exporter.write_allowed) is False
@@ -141,18 +136,16 @@ def files_unchanged():
 
     # Change some settings. Warnings should be printed in every case.
     clm_solver = clm.Solver.CLM
-    clm_solver.Input.Timing.StartYear = '2020'
+    clm_solver.Input.Timing.StartYear = "2020"
     assert was_called(exporter, warning_func, exporter.write_allowed) is True
 
     # Prove the files were not written to
     assert files_unchanged()
 
     # Re-import the files to erase history
-    CLMImporter(clm) \
-        .set_default_land_names() \
-        .files()
+    CLMImporter(clm).set_default_land_names().files()
 
-    clm_solver.Vegetation.Map.Sand.Type = 'Constant'
+    clm_solver.Vegetation.Map.Sand.Type = "Constant"
     clm_solver.Vegetation.Map.Sand.Value = 0.3
     assert was_called(exporter, warning_func, exporter.write_allowed) is True
 
@@ -160,9 +153,7 @@ def files_unchanged():
     assert files_unchanged()
 
     # Re-import the files to erase history
-    CLMImporter(clm) \
-        .set_default_land_names() \
-        .files()
+    CLMImporter(clm).set_default_land_names().files()
 
     clm_solver.Vegetation.Parameters.forest_en.WaterType = 5
     assert was_called(exporter, warning_func, exporter.write_allowed) is True
@@ -171,9 +162,7 @@ def files_unchanged():
     assert files_unchanged()
 
     # Re-import the files to erase history
-    CLMImporter(clm) \
-        .set_default_land_names() \
-        .files()
+    CLMImporter(clm).set_default_land_names().files()
     # Should not be called this time
     assert was_called(exporter, warning_func, exporter.write_allowed) is False
 
@@ -192,7 +181,7 @@ def assert_exception(func, *args, **kwargs):
     except exc:
         return
     else:
-        sys.exit(f'{exc} did not occur')
+        sys.exit(f"{exc} did not occur")
 
 
 def was_called(obj, func_name, run_func, *args, **kwargs):
@@ -228,8 +217,8 @@ def driver_file_written_times():
     }
 
 
-if __name__ == '__main__':
-    clm = Run('clm', __file__)
+if __name__ == "__main__":
+    clm = Run("clm", __file__)
 
     # These are required to write out the map
     clm.ComputationalGrid.NX = 5
diff --git a/test/python/new_features/clm_veg_mapping.py b/test/python/new_features/clm_veg_mapping.py
index 9c8aa755e..37a62816c 100644
--- a/test/python/new_features/clm_veg_mapping.py
+++ b/test/python/new_features/clm_veg_mapping.py
@@ -11,30 +11,32 @@
 
 clm = Run("clm", __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational grid keys
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 clm.ComputationalGrid.Lower.X = 0.0
 clm.ComputationalGrid.Lower.Y = 0.0
 clm.ComputationalGrid.Lower.Z = 0.0
 
-clm.ComputationalGrid.DX = 1000.
-clm.ComputationalGrid.DY = 1000.
+clm.ComputationalGrid.DX = 1000.0
+clm.ComputationalGrid.DY = 1000.0
 clm.ComputationalGrid.DZ = 0.5
 
 clm.ComputationalGrid.NX = 5
 clm.ComputationalGrid.NY = 5
 clm.ComputationalGrid.NZ = 10
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Testing clm data key setting
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # TODO: add to DomainBuilder
-clm.Solver.CLM.Vegetation.Parameters.LandNames = 'forest_en forest_eb forest_dn forest_db'
+clm.Solver.CLM.Vegetation.Parameters.LandNames = (
+    "forest_en forest_eb forest_dn forest_db"
+)
 
-clm.Solver.CLM.Vegetation.Map.Latitude.Type = 'Linear'
+clm.Solver.CLM.Vegetation.Map.Latitude.Type = "Linear"
 clm.Solver.CLM.Vegetation.Map.Latitude.Min = 34.750
 clm.Solver.CLM.Vegetation.Map.Latitude.Max = 35.750
 
@@ -42,46 +44,46 @@
 # clm.Solver.CLM.Vegetation.Map.Longitude.Type = 'PFBFile'
 # clm.Solver.CLM.Vegetation.Map.Longitude.FileName = 'longitude_mapping.pfb'
 
-clm.Solver.CLM.Vegetation.Map.Longitude.Type = 'Linear'
+clm.Solver.CLM.Vegetation.Map.Longitude.Type = "Linear"
 clm.Solver.CLM.Vegetation.Map.Longitude.Min = -99.00
 clm.Solver.CLM.Vegetation.Map.Longitude.Max = -98.00
 
-clm.Solver.CLM.Vegetation.Map.Sand.Type = 'Constant'
+clm.Solver.CLM.Vegetation.Map.Sand.Type = "Constant"
 clm.Solver.CLM.Vegetation.Map.Sand.Value = 0.16
 
-clm.Solver.CLM.Vegetation.Map.Clay.Type = 'Constant'
+clm.Solver.CLM.Vegetation.Map.Clay.Type = "Constant"
 clm.Solver.CLM.Vegetation.Map.Clay.Value = 0.265
 
-clm.Solver.CLM.Vegetation.Map.Color.Type = 'Constant'
+clm.Solver.CLM.Vegetation.Map.Color.Type = "Constant"
 clm.Solver.CLM.Vegetation.Map.Color.Value = 2
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Setting land use fractions
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-clm.Solver.CLM.Vegetation.Map.LandFrac.forest_en.Type = 'Constant'
+clm.Solver.CLM.Vegetation.Map.LandFrac.forest_en.Type = "Constant"
 clm.Solver.CLM.Vegetation.Map.LandFrac.forest_en.Value = 0.0
 
 forest_eb_mat = np.zeros((clm.ComputationalGrid.NX, clm.ComputationalGrid.NY))
 forest_eb_mat[1, :] = 1.0
-file_name = 'forest_eb_mat.pfb'
+file_name = "forest_eb_mat.pfb"
 write_pfb(get_absolute_path(file_name), forest_eb_mat)
 
-clm.Solver.CLM.Vegetation.Map.LandFrac.forest_eb.Type = 'PFBFile'
+clm.Solver.CLM.Vegetation.Map.LandFrac.forest_eb.Type = "PFBFile"
 clm.Solver.CLM.Vegetation.Map.LandFrac.forest_eb.FileName = file_name
 
-clm.Solver.CLM.Vegetation.Map.LandFrac.forest_dn.Type = 'Constant'
+clm.Solver.CLM.Vegetation.Map.LandFrac.forest_dn.Type = "Constant"
 clm.Solver.CLM.Vegetation.Map.LandFrac.forest_dn.Value = 0.5
 
-clm.Solver.CLM.Vegetation.Map.LandFrac.forest_db.Type = 'Constant'
+clm.Solver.CLM.Vegetation.Map.LandFrac.forest_db.Type = "Constant"
 clm.Solver.CLM.Vegetation.Map.LandFrac.forest_db.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Testing clm data reader for veg mapping
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # Reading drv_vegm.dat into 3D array
-vegm_data = read_clm('../../tcl/clm/drv_vegm.dat', type='vegm')
+vegm_data = read_clm("../../tcl/clm/drv_vegm.dat", type="vegm")
 if not vegm_data[1, 1, 14] == 1:
     sys.exit(1)
 
@@ -89,9 +91,8 @@
 # Testing clm data writers
 # ---------------------------------------------------------
 
-path = get_absolute_path('drv_vegm.dat')
+path = get_absolute_path("drv_vegm.dat")
 if Path(path).exists():
     Path(path).unlink()
 
-CLMExporter(clm) \
-    .write_map()
+CLMExporter(clm).write_map()
diff --git a/test/python/new_features/default_db.py b/test/python/new_features/default_db.py
index 0fde8070d..8f365847c 100644
--- a/test/python/new_features/default_db.py
+++ b/test/python/new_features/default_db.py
@@ -1,7 +1,7 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Testing loading of default database and adding run name argument to both the
 # initial class definition and the apply
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 import sys
 
@@ -13,17 +13,17 @@
 
 db_test.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set Processor topology
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 db_test.Process.Topology.P = 1
 db_test.Process.Topology.Q = 1
 db_test.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 db_test.ComputationalGrid.Lower.X = 0.0
 db_test.ComputationalGrid.Lower.Y = 0.0
@@ -37,22 +37,22 @@
 db_test.ComputationalGrid.NY = 41
 db_test.ComputationalGrid.NZ = 50
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-db_test.GeomInput.Names = 'box_input indi_input'
+db_test.GeomInput.Names = "box_input indi_input"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-db_test.GeomInput.box_input.InputType = 'Box'
-db_test.GeomInput.box_input.GeomName = 'domain'
+db_test.GeomInput.box_input.InputType = "Box"
+db_test.GeomInput.box_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 db_test.Geom.domain.Lower.X = 0.0
 db_test.Geom.domain.Lower.Y = 0.0
@@ -61,15 +61,17 @@
 db_test.Geom.domain.Upper.X = 41000.0
 db_test.Geom.domain.Upper.Y = 41000.0
 db_test.Geom.domain.Upper.Z = 100.0
-db_test.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+db_test.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Indicator Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-db_test.GeomInput.indi_input.InputType = 'IndicatorField'
-db_test.GeomInput.indi_input.GeomNames = 's1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 g1 g2 g3 g4 g5 g6 g7 g8'
-db_test.Geom.indi_input.FileName = 'IndicatorFile_Gleeson.50z.pfb'
+db_test.GeomInput.indi_input.InputType = "IndicatorField"
+db_test.GeomInput.indi_input.GeomNames = (
+    "s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 g1 g2 g3 g4 g5 g6 g7 g8"
+)
+db_test.Geom.indi_input.FileName = "IndicatorFile_Gleeson.50z.pfb"
 
 db_test.GeomInput.s1.Value = 1
 db_test.GeomInput.s2.Value = 2
@@ -93,51 +95,46 @@
 db_test.GeomInput.g7.Value = 27
 db_test.GeomInput.g8.Value = 28
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Reading subsurface properties from database
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 subsurf_mapping = {
-  'bedrock_1': ['domain', 'g2'],
-  'sand': 's1',
-  'loamy_sand': 's2',
-  'sandy_loam': 's3',
-  'silt_loam': 's4',
-  'silt': 's5',
-  'sandy_clay': 's6',
-  'loam': 's7',
-  'sandy_clay_loam': 's8',
-  'silty_clay_loam': 's9',
-  'clay_loam': 's10',
-  'silty_clay': 's11',
-  'clay': 's12',
-  'organic': 's13',
-  'sil_sedimentary': 'g1',
-  'bedrock_2': 'g3',
-  'crystalline': 'g4',
-  'fg_unconsolidated': 'g5',
-  'unconsolidated': 'g6',
-  'cg_sil_sedimentary': 'g7',
-  'carbonate': 'g8'
+    "bedrock_1": ["domain", "g2"],
+    "sand": "s1",
+    "loamy_sand": "s2",
+    "sandy_loam": "s3",
+    "silt_loam": "s4",
+    "silt": "s5",
+    "sandy_clay": "s6",
+    "loam": "s7",
+    "sandy_clay_loam": "s8",
+    "silty_clay_loam": "s9",
+    "clay_loam": "s10",
+    "silty_clay": "s11",
+    "clay": "s12",
+    "organic": "s13",
+    "sil_sedimentary": "g1",
+    "bedrock_2": "g3",
+    "crystalline": "g4",
+    "fg_unconsolidated": "g5",
+    "unconsolidated": "g6",
+    "cg_sil_sedimentary": "g7",
+    "carbonate": "g8",
 }
 
 # cloning run object to test optional run assignment
-db_test_2 = db_test.clone('db_test_2')
+db_test_2 = db_test.clone("db_test_2")
 
 # standard inline procedure calls in Python
-SubsurfacePropertiesBuilder(db_test)\
-  .load_default_properties() \
-  .assign(mapping=subsurf_mapping) \
-  .apply() \
-  .print_as_table()
+SubsurfacePropertiesBuilder(db_test).load_default_properties().assign(
+    mapping=subsurf_mapping
+).apply().print_as_table()
 
 # testing optional run assignment in apply
-SubsurfacePropertiesBuilder()\
-  .load_default_properties() \
-  .assign(mapping=subsurf_mapping) \
-  .assign('bedrock_2', 'g4') \
-  .apply(db_test_2) \
-  .print_as_table()
+SubsurfacePropertiesBuilder().load_default_properties().assign(
+    mapping=subsurf_mapping
+).assign("bedrock_2", "g4").apply(db_test_2).print_as_table()
 
 # or you can assign the properties one by one
 # SubsurfacePropertiesBuilder()\
@@ -149,160 +146,160 @@
 #   .print_as_table()
 
 
-db_test.Perm.TensorType = 'TensorByGeom'
-db_test.Geom.Perm.TensorByGeom.Names = 'domain'
+db_test.Perm.TensorType = "TensorByGeom"
+db_test.Geom.Perm.TensorByGeom.Names = "domain"
 db_test.Geom.domain.Perm.TensorValX = 1.0
 db_test.Geom.domain.Perm.TensorValY = 1.0
 db_test.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-db_test.SpecificStorage.Type = 'Constant'
-db_test.SpecificStorage.GeomNames = 'domain'
+db_test.SpecificStorage.Type = "Constant"
+db_test.SpecificStorage.GeomNames = "domain"
 db_test.Geom.domain.SpecificStorage.Value = 1.0e-5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-db_test.Phase.Names = 'water'
-db_test.Phase.water.Density.Type = 'Constant'
+db_test.Phase.Names = "water"
+db_test.Phase.water.Density.Type = "Constant"
 db_test.Phase.water.Density.Value = 1.0
-db_test.Phase.water.Viscosity.Type = 'Constant'
+db_test.Phase.water.Viscosity.Type = "Constant"
 db_test.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-db_test.Contaminants.Names = ''
+db_test.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 db_test.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Timing (time units is set by units of permeability)
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 db_test.TimingInfo.BaseUnit = 1.0
 db_test.TimingInfo.StartCount = 0
 db_test.TimingInfo.StartTime = 0.0
 db_test.TimingInfo.StopTime = 1000.0
 db_test.TimingInfo.DumpInterval = 1.0
-db_test.TimeStep.Type = 'Constant'
+db_test.TimeStep.Type = "Constant"
 db_test.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-db_test.Domain.GeomName = 'domain'
+db_test.Domain.GeomName = "domain"
 
-#----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 # Mobility
-#----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 
-db_test.Phase.water.Mobility.Type = 'Constant'
+db_test.Phase.water.Mobility.Type = "Constant"
 db_test.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-db_test.Wells.Names = ''
+db_test.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-db_test.Cycle.Names = 'constant rainrec'
-db_test.Cycle.constant.Names = 'alltime'
+db_test.Cycle.Names = "constant rainrec"
+db_test.Cycle.constant.Names = "alltime"
 db_test.Cycle.constant.alltime.Length = 1
 db_test.Cycle.constant.Repeat = -1
 
-db_test.Cycle.rainrec.Names = 'rain rec'
+db_test.Cycle.rainrec.Names = "rain rec"
 db_test.Cycle.rainrec.rain.Length = 10
 db_test.Cycle.rainrec.rec.Length = 150
 db_test.Cycle.rainrec.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 db_test.BCPressure.PatchNames = db_test.Geom.domain.Patches
 
-db_test.Patch.x_lower.BCPressure.Type = 'FluxConst'
-db_test.Patch.x_lower.BCPressure.Cycle = 'constant'
+db_test.Patch.x_lower.BCPressure.Type = "FluxConst"
+db_test.Patch.x_lower.BCPressure.Cycle = "constant"
 db_test.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-db_test.Patch.y_lower.BCPressure.Type = 'FluxConst'
-db_test.Patch.y_lower.BCPressure.Cycle = 'constant'
+db_test.Patch.y_lower.BCPressure.Type = "FluxConst"
+db_test.Patch.y_lower.BCPressure.Cycle = "constant"
 db_test.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-db_test.Patch.z_lower.BCPressure.Type = 'FluxConst'
-db_test.Patch.z_lower.BCPressure.Cycle = 'constant'
+db_test.Patch.z_lower.BCPressure.Type = "FluxConst"
+db_test.Patch.z_lower.BCPressure.Cycle = "constant"
 db_test.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-db_test.Patch.x_upper.BCPressure.Type = 'FluxConst'
-db_test.Patch.x_upper.BCPressure.Cycle = 'constant'
+db_test.Patch.x_upper.BCPressure.Type = "FluxConst"
+db_test.Patch.x_upper.BCPressure.Cycle = "constant"
 db_test.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-db_test.Patch.y_upper.BCPressure.Type = 'FluxConst'
-db_test.Patch.y_upper.BCPressure.Cycle = 'constant'
+db_test.Patch.y_upper.BCPressure.Type = "FluxConst"
+db_test.Patch.y_upper.BCPressure.Cycle = "constant"
 db_test.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-db_test.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-db_test.Patch.z_upper.BCPressure.Cycle = 'rainrec'
+db_test.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+db_test.Patch.z_upper.BCPressure.Cycle = "rainrec"
 db_test.Patch.z_upper.BCPressure.rain.Value = -0.1
 db_test.Patch.z_upper.BCPressure.rec.Value = 0.0000
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Topo slopes in x-direction
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-db_test.TopoSlopesX.Type = 'PFBFile'
-db_test.TopoSlopesX.GeomNames = 'domain'
-db_test.TopoSlopesX.FileName = 'LW.slopex.pfb'
+db_test.TopoSlopesX.Type = "PFBFile"
+db_test.TopoSlopesX.GeomNames = "domain"
+db_test.TopoSlopesX.FileName = "LW.slopex.pfb"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Topo slopes in y-direction
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-db_test.TopoSlopesY.Type = 'PFBFile'
-db_test.TopoSlopesY.GeomNames = 'domain'
-db_test.TopoSlopesY.FileName = 'LW.slopey.pfb'
+db_test.TopoSlopesY.Type = "PFBFile"
+db_test.TopoSlopesY.GeomNames = "domain"
+db_test.TopoSlopesY.FileName = "LW.slopey.pfb"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mannings coefficient
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-db_test.Mannings.Type = 'Constant'
-db_test.Mannings.GeomNames = 'domain'
+db_test.Mannings.Type = "Constant"
+db_test.Mannings.GeomNames = "domain"
 db_test.Mannings.Geom.domain.Value = 5.52e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-db_test.PhaseSources.water.Type = 'Constant'
-db_test.PhaseSources.water.GeomNames = 'domain'
+db_test.PhaseSources.water.Type = "Constant"
+db_test.PhaseSources.water.GeomNames = "domain"
 db_test.PhaseSources.water.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-db_test.ICPressure.Type = 'PFBFile'
-db_test.ICPressure.GeomNames = 'domain'
-db_test.Geom.domain.ICPressure.RefPatch = 'z_upper'
-db_test.Geom.domain.ICPressure.FileName = 'press.init.pfb'
+db_test.ICPressure.Type = "PFBFile"
+db_test.ICPressure.GeomNames = "domain"
+db_test.Geom.domain.ICPressure.RefPatch = "z_upper"
+db_test.Geom.domain.ICPressure.FileName = "press.init.pfb"
 
-#----------------------------------------------------------------
+# ----------------------------------------------------------------
 # Outputs
 # ---------------------------------------------------------------
 
@@ -311,35 +308,35 @@
 db_test.Solver.PrintSaturation = True
 db_test.Solver.PrintMask = True
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-db_test.KnownSolution = 'NoKnownSolution'
+db_test.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 # ParFlow Solution
-db_test.Solver = 'Richards'
+db_test.Solver = "Richards"
 db_test.Solver.TerrainFollowingGrid = True
 db_test.Solver.Nonlinear.VariableDz = False
 
 db_test.Solver.MaxIter = 25000
-db_test.Solver.Drop = 1E-20
-db_test.Solver.AbsTol = 1E-8
+db_test.Solver.Drop = 1e-20
+db_test.Solver.AbsTol = 1e-8
 db_test.Solver.MaxConvergenceFailures = 8
 db_test.Solver.Nonlinear.MaxIter = 80
 db_test.Solver.Nonlinear.ResidualTol = 1e-6
 
-db_test.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+db_test.Solver.Nonlinear.EtaChoice = "EtaConstant"
 db_test.Solver.Nonlinear.EtaValue = 0.001
 db_test.Solver.Nonlinear.UseJacobian = True
 db_test.Solver.Nonlinear.DerivativeEpsilon = 1e-16
 db_test.Solver.Nonlinear.StepTol = 1e-30
-db_test.Solver.Nonlinear.Globalization = 'LineSearch'
+db_test.Solver.Nonlinear.Globalization = "LineSearch"
 db_test.Solver.Linear.KrylovDimension = 70
 db_test.Solver.Linear.MaxRestarts = 2
 
-db_test.Solver.Linear.Preconditioner = 'PFMG'
+db_test.Solver.Linear.Preconditioner = "PFMG"
diff --git a/test/python/new_features/domain_builder.py b/test/python/new_features/domain_builder.py
index 49507f67f..af72fbf04 100644
--- a/test/python/new_features/domain_builder.py
+++ b/test/python/new_features/domain_builder.py
@@ -1,8 +1,8 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  This runs the basic default_richards test case.
 #  This run, as written in this input file, should take
 #  3 nonlinear iterations.
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 import sys
 
@@ -10,17 +10,17 @@
 from parflow.tools.fs import get_absolute_path
 from parflow.tools.builders import DomainBuilder
 
-drich = Run('drich', __file__)
+drich = Run("drich", __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Process.Topology.P = 1
 drich.Process.Topology.Q = 1
 drich.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.ComputationalGrid.Lower.X = -10.0
 drich.ComputationalGrid.Lower.Y = 10.0
@@ -34,148 +34,130 @@
 drich.ComputationalGrid.NY = 10
 drich.ComputationalGrid.NZ = 8
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Time Cycles
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.Cycle.Names = 'constant'
+drich.Cycle.Names = "constant"
 drich.Cycle.constant.Names = "alltime"
 drich.Cycle.constant.alltime.Length = 1
 drich.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.TimingInfo.BaseUnit = 1.0
 drich.TimingInfo.StartCount = 0
 drich.TimingInfo.StartTime = 0.0
 drich.TimingInfo.StopTime = 0.010
 drich.TimingInfo.DumpInterval = -1
-drich.TimeStep.Type = 'Constant'
+drich.TimeStep.Type = "Constant"
 drich.TimeStep.Value = 0.001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain setup using builder
-#---------------------------------------------------------
-
-DomainBuilder(drich) \
-    .water('background') \
-    .variably_saturated() \
-    .box_domain('domain_input', 'domain',
-                bounds=[
-                  -10.0, 150.0,   # x range
-                  10.0, 170.0,    # y range
-                  1.0, 9.0        # z range
-                ],
-                patches="left right front back bottom top") \
-    .box_domain('background_input', 'background',
-                bounds=[
-                  -99999999.0, 99999999.0,
-                  -99999999.0, 99999999.0,
-                  -99999999.0, 99999999.0
-                ]) \
-    .box_domain('source_region_input', 'source_region',
-                bounds=[
-                  65.56, 74.44,
-                  79.34, 89.99,
-                  4.5, 5.5
-                ]) \
-    .box_domain('concen_region_input', 'concen_region',
-                bounds=[
-                  60.0, 80.0,
-                  80.0, 100.0,
-                  4.0, 6.0
-                ]) \
-    .slopes_mannings('domain', slope_x=0.0, slope_y=0.0, mannings=0.0) \
-    .homogeneous_subsurface('domain',
-                            specific_storage=1.0e-4,
-                            rel_perm={
-                              'Type': 'VanGenuchten',
-                              'Alpha': 0.005,
-                              'N': 2.0
-                            },
-                            saturation={
-                              'Type': 'VanGenuchten',
-                              'Alpha': 0.005,         # optional, uses RelPerm.Alpha if missing
-                              'N': 2.0,               # optional, uses RelPerm.N if missing
-                              'SRes': 0.2,
-                              'SSat': 0.99
-                            }
-                            ) \
-    .homogeneous_subsurface('background',
-                            perm=4.0,
-                            porosity=1.0,
-                            isotropic=True
-                            ) \
-    .zero_flux(patches='front back bottom top',
-               cycle_name='constant',
-               interval_name='alltime')
-
-#-----------------------------------------------------------------------------
+# ---------------------------------------------------------
+
+DomainBuilder(drich).water("background").variably_saturated().box_domain(
+    "domain_input",
+    "domain",
+    bounds=[-10.0, 150.0, 10.0, 170.0, 1.0, 9.0],  # x range  # y range  # z range
+    patches="left right front back bottom top",
+).box_domain(
+    "background_input",
+    "background",
+    bounds=[-99999999.0, 99999999.0, -99999999.0, 99999999.0, -99999999.0, 99999999.0],
+).box_domain(
+    "source_region_input",
+    "source_region",
+    bounds=[65.56, 74.44, 79.34, 89.99, 4.5, 5.5],
+).box_domain(
+    "concen_region_input", "concen_region", bounds=[60.0, 80.0, 80.0, 100.0, 4.0, 6.0]
+).slopes_mannings(
+    "domain", slope_x=0.0, slope_y=0.0, mannings=0.0
+).homogeneous_subsurface(
+    "domain",
+    specific_storage=1.0e-4,
+    rel_perm={"Type": "VanGenuchten", "Alpha": 0.005, "N": 2.0},
+    saturation={
+        "Type": "VanGenuchten",
+        "Alpha": 0.005,  # optional, uses RelPerm.Alpha if missing
+        "N": 2.0,  # optional, uses RelPerm.N if missing
+        "SRes": 0.2,
+        "SSat": 0.99,
+    },
+).homogeneous_subsurface(
+    "background", perm=4.0, porosity=1.0, isotropic=True
+).zero_flux(
+    patches="front back bottom top", cycle_name="constant", interval_name="alltime"
+)
+
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Wells.Names = ''
+drich.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.BCPressure.PatchNames = "left right front back bottom top"
 
-drich.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
+drich.Patch.left.BCPressure.Type = "DirEquilRefPatch"
 drich.Patch.left.BCPressure.Cycle = "constant"
-drich.Patch.left.BCPressure.RefGeom = 'domain'
-drich.Patch.left.BCPressure.RefPatch = 'bottom'
+drich.Patch.left.BCPressure.RefGeom = "domain"
+drich.Patch.left.BCPressure.RefPatch = "bottom"
 drich.Patch.left.BCPressure.alltime.Value = 5.0
 
-drich.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
+drich.Patch.right.BCPressure.Type = "DirEquilRefPatch"
 drich.Patch.right.BCPressure.Cycle = "constant"
-drich.Patch.right.BCPressure.RefGeom = 'domain'
-drich.Patch.right.BCPressure.RefPatch = 'bottom'
+drich.Patch.right.BCPressure.RefGeom = "domain"
+drich.Patch.right.BCPressure.RefPatch = "bottom"
 drich.Patch.right.BCPressure.alltime.Value = 3.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.ICPressure.Type = 'HydroStaticPatch'
-drich.ICPressure.GeomNames = 'domain'
+drich.ICPressure.Type = "HydroStaticPatch"
+drich.ICPressure.GeomNames = "domain"
 drich.Geom.domain.ICPressure.Value = 3.0
-drich.Geom.domain.ICPressure.RefGeom = 'domain'
-drich.Geom.domain.ICPressure.RefPatch = 'bottom'
+drich.Geom.domain.ICPressure.RefGeom = "domain"
+drich.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.KnownSolution = 'NoKnownSolution'
+drich.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run ParFlow
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.validate()
 
-drich.write(file_format='yaml')
+drich.write(file_format="yaml")
 
-with open(get_absolute_path('drich.yaml')) as new, \
-    open(get_absolute_path('../../correct_output/domain_builder.yaml.ref')) as ref:
-  if new.read() == ref.read():
-    print('Success we have the same file')
-  else:
-    print('Files are different')
-    sys.exit(1)
+with open(get_absolute_path("drich.yaml")) as new, open(
+    get_absolute_path("../../correct_output/domain_builder.yaml.ref")
+) as ref:
+    if new.read() == ref.read():
+        print("Success we have the same file")
+    else:
+        print("Files are different")
+        sys.exit(1)
diff --git a/test/python/new_features/domain_drich_wells.py b/test/python/new_features/domain_drich_wells.py
index 074d09773..0b497b564 100644
--- a/test/python/new_features/domain_drich_wells.py
+++ b/test/python/new_features/domain_drich_wells.py
@@ -1,8 +1,8 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 #  This runs the basic default_richards test case.
 #  This run, as written in this input file, should take
 #  3 nonlinear iterations.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 import sys
 
@@ -12,7 +12,7 @@
 
 drich = Run("drich_wells_build", __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.FileVersion = 4
 
@@ -20,9 +20,9 @@
 drich.Process.Topology.Q = 1
 drich.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.ComputationalGrid.Lower.X = -10.0
 drich.ComputationalGrid.Lower.Y = 10.0
@@ -36,22 +36,24 @@
 drich.ComputationalGrid.NY = 10
 drich.ComputationalGrid.NZ = 8
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.Names = 'domain_input background_input source_region_input concen_region_input'
+drich.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.domain_input.InputType = 'Box'
-drich.GeomInput.domain_input.GeomName = 'domain'
+drich.GeomInput.domain_input.InputType = "Box"
+drich.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Geom.domain.Lower.X = -10.0
 drich.Geom.domain.Lower.Y = 10.0
@@ -61,18 +63,18 @@
 drich.Geom.domain.Upper.Y = 170.0
 drich.Geom.domain.Upper.Z = 9.0
 
-drich.Geom.domain.Patches = 'left right front back bottom top'
+drich.Geom.domain.Patches = "left right front back bottom top"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.background_input.InputType = 'Box'
-drich.GeomInput.background_input.GeomName = 'background'
+drich.GeomInput.background_input.InputType = "Box"
+drich.GeomInput.background_input.GeomName = "background"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Geom.background.Lower.X = -99999999.0
 drich.Geom.background.Lower.Y = -99999999.0
@@ -82,16 +84,16 @@
 drich.Geom.background.Upper.Y = 99999999.0
 drich.Geom.background.Upper.Z = 99999999.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.source_region_input.InputType = 'Box'
-drich.GeomInput.source_region_input.GeomName = 'source_region'
+drich.GeomInput.source_region_input.InputType = "Box"
+drich.GeomInput.source_region_input.GeomName = "source_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Geom.source_region.Lower.X = 65.56
 drich.Geom.source_region.Lower.Y = 79.34
@@ -101,16 +103,16 @@
 drich.Geom.source_region.Upper.Y = 89.99
 drich.Geom.source_region.Upper.Z = 5.5
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.concen_region_input.InputType = 'Box'
-drich.GeomInput.concen_region_input.GeomName = 'concen_region'
+drich.GeomInput.concen_region_input.InputType = "Box"
+drich.GeomInput.concen_region_input.GeomName = "concen_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Geom.concen_region.Lower.X = 60.0
 drich.Geom.concen_region.Lower.Y = 80.0
@@ -120,125 +122,134 @@
 drich.Geom.concen_region.Upper.Y = 100.0
 drich.Geom.concen_region.Upper.Z = 6.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Geom.Perm.Names = 'background'
+drich.Geom.Perm.Names = "background"
 
-drich.Geom.background.Perm.Type = 'Constant'
+drich.Geom.background.Perm.Type = "Constant"
 drich.Geom.background.Perm.Value = 4.0
 
-drich.Perm.TensorType = 'TensorByGeom'
+drich.Perm.TensorType = "TensorByGeom"
 
-drich.Geom.Perm.TensorByGeom.Names = 'background'
+drich.Geom.Perm.TensorByGeom.Names = "background"
 
 drich.Geom.background.Perm.TensorValX = 1.0
 drich.Geom.background.Perm.TensorValY = 1.0
 drich.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.SpecificStorage.Type = 'Constant'
-drich.SpecificStorage.GeomNames = 'domain'
+drich.SpecificStorage.Type = "Constant"
+drich.SpecificStorage.GeomNames = "domain"
 drich.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Phase.Names = 'water'
+drich.Phase.Names = "water"
 
-drich.Phase.water.Density.Type = 'Constant'
+drich.Phase.water.Density.Type = "Constant"
 drich.Phase.water.Density.Value = 1.0
 
-drich.Phase.water.Viscosity.Type = 'Constant'
+drich.Phase.water.Viscosity.Type = "Constant"
 drich.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Contaminants.Names = ''
+drich.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Geom.Retardation.GeomNames = ''
+drich.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.TimingInfo.BaseUnit = 1.0
 drich.TimingInfo.StartCount = 0
 drich.TimingInfo.StartTime = 0.0
 drich.TimingInfo.StopTime = 0.010
 drich.TimingInfo.DumpInterval = -1
-drich.TimeStep.Type = 'Constant'
+drich.TimeStep.Type = "Constant"
 drich.TimeStep.Value = 0.001
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Geom.Porosity.GeomNames = 'background'
+drich.Geom.Porosity.GeomNames = "background"
 
-drich.Geom.background.Porosity.Type = 'Constant'
+drich.Geom.background.Porosity.Type = "Constant"
 drich.Geom.background.Porosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
-drich.Domain.GeomName = 'domain'
+# -----------------------------------------------------------------------------
+drich.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Phase.RelPerm.Type = 'VanGenuchten'
-drich.Phase.RelPerm.GeomNames = 'domain'
+drich.Phase.RelPerm.Type = "VanGenuchten"
+drich.Phase.RelPerm.GeomNames = "domain"
 drich.Geom.domain.RelPerm.Alpha = 0.005
 drich.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.Phase.Saturation.Type = 'VanGenuchten'
-drich.Phase.Saturation.GeomNames = 'domain'
+drich.Phase.Saturation.Type = "VanGenuchten"
+drich.Phase.Saturation.GeomNames = "domain"
 drich.Geom.domain.Saturation.Alpha = 0.005
 drich.Geom.domain.Saturation.N = 2.0
 drich.Geom.domain.Saturation.SRes = 0.2
 drich.Geom.domain.Saturation.SSat = 0.99
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Cycle.Names = 'constant'
-drich.Cycle.constant.Names = 'alltime'
+drich.Cycle.Names = "constant"
+drich.Cycle.constant.Names = "alltime"
 drich.Cycle.constant.alltime.Length = 1
 drich.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
-
-DomainBuilder(drich) \
-    .well(name='pumping_well', type='Pressure', x=0, y=80,
-                  z_upper=3.0, z_lower=2.00, cycle_name='constant',
-                  interval_name='alltime', saturation=1.0, phase='water',
-                  hydrostatic_pressure=0.5, value=None)
+# -----------------------------------------------------------------------------
+
+DomainBuilder(drich).well(
+    name="pumping_well",
+    type="Pressure",
+    x=0,
+    y=80,
+    z_upper=3.0,
+    z_lower=2.00,
+    cycle_name="constant",
+    interval_name="alltime",
+    saturation=1.0,
+    phase="water",
+    hydrostatic_pressure=0.5,
+    value=None,
+)
 
 # drich.Wells.Names = 'pumping_well'
 # drich.Wells.pumping_well.InputType = 'Vertical'
@@ -253,117 +264,118 @@
 # drich.Wells.pumping_well.alltime.Pressure.Value = 0.5
 # drich.Wells.pumping_well.alltime.Saturation.water.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.BCPressure.PatchNames = 'left right front back bottom top'
+drich.BCPressure.PatchNames = "left right front back bottom top"
 
-drich.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-drich.Patch.left.BCPressure.Cycle = 'constant'
-drich.Patch.left.BCPressure.RefGeom = 'domain'
-drich.Patch.left.BCPressure.RefPatch = 'bottom'
+drich.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+drich.Patch.left.BCPressure.Cycle = "constant"
+drich.Patch.left.BCPressure.RefGeom = "domain"
+drich.Patch.left.BCPressure.RefPatch = "bottom"
 drich.Patch.left.BCPressure.alltime.Value = 5.0
 
-drich.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-drich.Patch.right.BCPressure.Cycle = 'constant'
-drich.Patch.right.BCPressure.RefGeom = 'domain'
-drich.Patch.right.BCPressure.RefPatch = 'bottom'
+drich.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+drich.Patch.right.BCPressure.Cycle = "constant"
+drich.Patch.right.BCPressure.RefGeom = "domain"
+drich.Patch.right.BCPressure.RefPatch = "bottom"
 drich.Patch.right.BCPressure.alltime.Value = 5.0
 
-drich.Patch.front.BCPressure.Type = 'FluxConst'
-drich.Patch.front.BCPressure.Cycle = 'constant'
+drich.Patch.front.BCPressure.Type = "FluxConst"
+drich.Patch.front.BCPressure.Cycle = "constant"
 drich.Patch.front.BCPressure.alltime.Value = 0.0
 
-drich.Patch.back.BCPressure.Type = 'FluxConst'
-drich.Patch.back.BCPressure.Cycle = 'constant'
+drich.Patch.back.BCPressure.Type = "FluxConst"
+drich.Patch.back.BCPressure.Cycle = "constant"
 drich.Patch.back.BCPressure.alltime.Value = 0.0
 
-drich.Patch.bottom.BCPressure.Type = 'FluxConst'
-drich.Patch.bottom.BCPressure.Cycle = 'constant'
+drich.Patch.bottom.BCPressure.Type = "FluxConst"
+drich.Patch.bottom.BCPressure.Cycle = "constant"
 drich.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-drich.Patch.top.BCPressure.Type = 'FluxConst'
-drich.Patch.top.BCPressure.Cycle = 'constant'
+drich.Patch.top.BCPressure.Type = "FluxConst"
+drich.Patch.top.BCPressure.Cycle = "constant"
 drich.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.TopoSlopesX.Type = 'Constant'
-drich.TopoSlopesX.GeomNames = 'domain'
+drich.TopoSlopesX.Type = "Constant"
+drich.TopoSlopesX.GeomNames = "domain"
 
 drich.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.TopoSlopesY.Type = 'Constant'
-drich.TopoSlopesY.GeomNames = 'domain'
+drich.TopoSlopesY.Type = "Constant"
+drich.TopoSlopesY.GeomNames = "domain"
 drich.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-drich.Mannings.Type = 'Constant'
-drich.Mannings.GeomNames = 'domain'
-drich.Mannings.Geom.domain.Value = 0.
+drich.Mannings.Type = "Constant"
+drich.Mannings.GeomNames = "domain"
+drich.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.ICPressure.Type = 'HydroStaticPatch'
-drich.ICPressure.GeomNames = 'domain'
+drich.ICPressure.Type = "HydroStaticPatch"
+drich.ICPressure.GeomNames = "domain"
 drich.Geom.domain.ICPressure.Value = 5.0
-drich.Geom.domain.ICPressure.RefGeom = 'domain'
-drich.Geom.domain.ICPressure.RefPatch = 'bottom'
+drich.Geom.domain.ICPressure.RefGeom = "domain"
+drich.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.PhaseSources.water.Type = 'Constant'
-drich.PhaseSources.water.GeomNames = 'background'
+drich.PhaseSources.water.Type = "Constant"
+drich.PhaseSources.water.GeomNames = "background"
 drich.PhaseSources.water.Geom.background.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.KnownSolution = 'NoKnownSolution'
+drich.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Solver = 'Richards'
+drich.Solver = "Richards"
 drich.Solver.MaxIter = 5
 
 drich.Solver.Nonlinear.MaxIter = 10
 drich.Solver.Nonlinear.ResidualTol = 1e-9
-drich.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+drich.Solver.Nonlinear.EtaChoice = "EtaConstant"
 drich.Solver.Nonlinear.EtaValue = 1e-5
 drich.Solver.Nonlinear.UseJacobian = True
 drich.Solver.Nonlinear.DerivativeEpsilon = 1e-2
 
 drich.Solver.Linear.KrylovDimension = 10
 
-drich.Solver.Linear.Preconditioner = 'MGSemi'
+drich.Solver.Linear.Preconditioner = "MGSemi"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-drich.write(file_format='yaml')
-
-with open(get_absolute_path('drich_wells_build.yaml')) as new, \
-      open(get_absolute_path('../../correct_output/drich_wells.yaml.ref')) as ref:
-  if new.read() == ref.read():
-      print('Success we have the same file')
-  else:
-      print('Files are different')
-      sys.exit(1)
+# -----------------------------------------------------------------------------
+
+drich.write(file_format="yaml")
+
+with open(get_absolute_path("drich_wells_build.yaml")) as new, open(
+    get_absolute_path("../../correct_output/drich_wells.yaml.ref")
+) as ref:
+    if new.read() == ref.read():
+        print("Success we have the same file")
+    else:
+        print("Files are different")
+        sys.exit(1)
diff --git a/test/python/new_features/enum_versioning.py b/test/python/new_features/enum_versioning.py
index 92fe72389..b52553d5d 100644
--- a/test/python/new_features/enum_versioning.py
+++ b/test/python/new_features/enum_versioning.py
@@ -1,7 +1,7 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Testing versioning of enumerated list domain for Python
 # library
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 import sys
 from parflow import Run
@@ -9,9 +9,9 @@
 
 overland_FlatICP = Run("overland_FlatICP", __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland_FlatICP.ComputationalGrid.Lower.X = 0.0
 overland_FlatICP.ComputationalGrid.Lower.Y = 0.0
@@ -23,23 +23,23 @@
 
 overland_FlatICP.ComputationalGrid.DX = 10.0
 overland_FlatICP.ComputationalGrid.DY = 10.0
-overland_FlatICP.ComputationalGrid.DZ = .05
+overland_FlatICP.ComputationalGrid.DZ = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland_FlatICP.GeomInput.Names = 'domaininput sourceinput'
+overland_FlatICP.GeomInput.Names = "domaininput sourceinput"
 
-overland_FlatICP.GeomInput.domaininput.GeomName = 'domain'
-overland_FlatICP.GeomInput.domaininput.InputType = 'Box'
+overland_FlatICP.GeomInput.domaininput.GeomName = "domain"
+overland_FlatICP.GeomInput.domaininput.InputType = "Box"
 
-overland_FlatICP.GeomInput.sourceinput.GeomName = 'icsource'
-overland_FlatICP.GeomInput.sourceinput.InputType = 'Box'
+overland_FlatICP.GeomInput.sourceinput.GeomName = "icsource"
+overland_FlatICP.GeomInput.sourceinput.InputType = "Box"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland_FlatICP.Geom.domain.Lower.X = 0.0
 overland_FlatICP.Geom.domain.Lower.Y = 0.0
@@ -48,7 +48,7 @@
 overland_FlatICP.Geom.domain.Upper.X = 100.0
 overland_FlatICP.Geom.domain.Upper.Y = 100.0
 overland_FlatICP.Geom.domain.Upper.Z = 0.05
-overland_FlatICP.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+overland_FlatICP.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
 overland_FlatICP.Geom.icsource.Lower.X = 40.0
 overland_FlatICP.Geom.icsource.Lower.Y = 40.0
@@ -58,309 +58,308 @@
 overland_FlatICP.Geom.icsource.Upper.Y = 60.0
 overland_FlatICP.Geom.icsource.Upper.Z = 0.05
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland_FlatICP.Geom.Perm.Names = 'domain'
+overland_FlatICP.Geom.Perm.Names = "domain"
 
 # Values in m/hour
-overland_FlatICP.Geom.domain.Perm.Type = 'Constant'
-overland_FlatICP.Geom.domain.Perm.Value = .000694
+overland_FlatICP.Geom.domain.Perm.Type = "Constant"
+overland_FlatICP.Geom.domain.Perm.Value = 0.000694
 
-overland_FlatICP.Perm.TensorType = 'TensorByGeom'
-overland_FlatICP.Geom.Perm.TensorByGeom.Names = 'domain'
+overland_FlatICP.Perm.TensorType = "TensorByGeom"
+overland_FlatICP.Geom.Perm.TensorByGeom.Names = "domain"
 
 overland_FlatICP.Geom.domain.Perm.TensorValX = 1.0
 overland_FlatICP.Geom.domain.Perm.TensorValY = 1.0
 overland_FlatICP.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland_FlatICP.SpecificStorage.Type = 'Constant'
-overland_FlatICP.SpecificStorage.GeomNames = 'domain'
+overland_FlatICP.SpecificStorage.Type = "Constant"
+overland_FlatICP.SpecificStorage.GeomNames = "domain"
 overland_FlatICP.Geom.domain.SpecificStorage.Value = 5.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland_FlatICP.Phase.Names = 'water'
+overland_FlatICP.Phase.Names = "water"
 
-overland_FlatICP.Phase.water.Density.Type = 'Constant'
+overland_FlatICP.Phase.water.Density.Type = "Constant"
 overland_FlatICP.Phase.water.Density.Value = 1.0
 
-overland_FlatICP.Phase.water.Viscosity.Type = 'Constant'
+overland_FlatICP.Phase.water.Viscosity.Type = "Constant"
 overland_FlatICP.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland_FlatICP.Contaminants.Names = ''
+overland_FlatICP.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland_FlatICP.Geom.Retardation.GeomNames = ''
+overland_FlatICP.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 overland_FlatICP.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 overland_FlatICP.TimingInfo.BaseUnit = 1.0
 overland_FlatICP.TimingInfo.StartCount = 0
 overland_FlatICP.TimingInfo.StartTime = 0.0
 overland_FlatICP.TimingInfo.StopTime = 300.0
 overland_FlatICP.TimingInfo.DumpInterval = 30.0
-overland_FlatICP.TimeStep.Type = 'Constant'
+overland_FlatICP.TimeStep.Type = "Constant"
 overland_FlatICP.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland_FlatICP.Geom.Porosity.GeomNames = 'domain'
-overland_FlatICP.Geom.domain.Porosity.Type = 'Constant'
+overland_FlatICP.Geom.Porosity.GeomNames = "domain"
+overland_FlatICP.Geom.domain.Porosity.Type = "Constant"
 overland_FlatICP.Geom.domain.Porosity.Value = 0.001
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland_FlatICP.Domain.GeomName = 'domain'
+overland_FlatICP.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland_FlatICP.Phase.RelPerm.Type = 'VanGenuchten'
-overland_FlatICP.Phase.RelPerm.GeomNames = 'domain'
+overland_FlatICP.Phase.RelPerm.Type = "VanGenuchten"
+overland_FlatICP.Phase.RelPerm.GeomNames = "domain"
 
 overland_FlatICP.Geom.domain.RelPerm.Alpha = 1.0
-overland_FlatICP.Geom.domain.RelPerm.N = 2.
+overland_FlatICP.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland_FlatICP.Phase.Saturation.Type = 'VanGenuchten'
-overland_FlatICP.Phase.Saturation.GeomNames = 'domain'
+overland_FlatICP.Phase.Saturation.Type = "VanGenuchten"
+overland_FlatICP.Phase.Saturation.GeomNames = "domain"
 
 overland_FlatICP.Geom.domain.Saturation.Alpha = 1.0
-overland_FlatICP.Geom.domain.Saturation.N = 2.
+overland_FlatICP.Geom.domain.Saturation.N = 2.0
 overland_FlatICP.Geom.domain.Saturation.SRes = 0.2
 overland_FlatICP.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland_FlatICP.Wells.Names = ''
+overland_FlatICP.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland_FlatICP.Cycle.Names = 'constant rainrec'
-overland_FlatICP.Cycle.constant.Names = 'alltime'
+overland_FlatICP.Cycle.Names = "constant rainrec"
+overland_FlatICP.Cycle.constant.Names = "alltime"
 overland_FlatICP.Cycle.constant.alltime.Length = 1
 overland_FlatICP.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-overland_FlatICP.Cycle.rainrec.Names = 'rain rec'
+overland_FlatICP.Cycle.rainrec.Names = "rain rec"
 overland_FlatICP.Cycle.rainrec.rain.Length = 200
 overland_FlatICP.Cycle.rainrec.rec.Length = 100
 overland_FlatICP.Cycle.rainrec.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 overland_FlatICP.BCPressure.PatchNames = overland_FlatICP.Geom.domain.Patches
 
-overland_FlatICP.Patch.x_lower.BCPressure.Type = 'FluxConst'
-overland_FlatICP.Patch.x_lower.BCPressure.Cycle = 'constant'
+overland_FlatICP.Patch.x_lower.BCPressure.Type = "FluxConst"
+overland_FlatICP.Patch.x_lower.BCPressure.Cycle = "constant"
 overland_FlatICP.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-overland_FlatICP.Patch.y_lower.BCPressure.Type = 'FluxConst'
-overland_FlatICP.Patch.y_lower.BCPressure.Cycle = 'constant'
+overland_FlatICP.Patch.y_lower.BCPressure.Type = "FluxConst"
+overland_FlatICP.Patch.y_lower.BCPressure.Cycle = "constant"
 overland_FlatICP.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-overland_FlatICP.Patch.z_lower.BCPressure.Type = 'FluxConst'
-overland_FlatICP.Patch.z_lower.BCPressure.Cycle = 'constant'
+overland_FlatICP.Patch.z_lower.BCPressure.Type = "FluxConst"
+overland_FlatICP.Patch.z_lower.BCPressure.Cycle = "constant"
 overland_FlatICP.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-overland_FlatICP.Patch.x_upper.BCPressure.Type = 'FluxConst'
-overland_FlatICP.Patch.x_upper.BCPressure.Cycle = 'constant'
+overland_FlatICP.Patch.x_upper.BCPressure.Type = "FluxConst"
+overland_FlatICP.Patch.x_upper.BCPressure.Cycle = "constant"
 overland_FlatICP.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-overland_FlatICP.Patch.y_upper.BCPressure.Type = 'FluxConst'
-overland_FlatICP.Patch.y_upper.BCPressure.Cycle = 'constant'
+overland_FlatICP.Patch.y_upper.BCPressure.Type = "FluxConst"
+overland_FlatICP.Patch.y_upper.BCPressure.Cycle = "constant"
 overland_FlatICP.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
 ## overland flow boundary condition with very heavy rainfall then slight ET
-overland_FlatICP.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-overland_FlatICP.Patch.z_upper.BCPressure.Cycle = 'rainrec'
+overland_FlatICP.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+overland_FlatICP.Patch.z_upper.BCPressure.Cycle = "rainrec"
 overland_FlatICP.Patch.z_upper.BCPressure.rain.Value = 0.0
 overland_FlatICP.Patch.z_upper.BCPressure.rec.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland_FlatICP.TopoSlopesX.Type = 'Constant'
-overland_FlatICP.TopoSlopesX.GeomNames = 'domain'
+overland_FlatICP.TopoSlopesX.Type = "Constant"
+overland_FlatICP.TopoSlopesX.GeomNames = "domain"
 overland_FlatICP.TopoSlopesX.Geom.domain.Value = 0.00
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland_FlatICP.TopoSlopesY.Type = 'Constant'
-overland_FlatICP.TopoSlopesY.GeomNames = 'domain'
+overland_FlatICP.TopoSlopesY.Type = "Constant"
+overland_FlatICP.TopoSlopesY.GeomNames = "domain"
 overland_FlatICP.TopoSlopesY.Geom.domain.Value = 0.00
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland_FlatICP.Mannings.Type = 'Constant'
-overland_FlatICP.Mannings.GeomNames = 'domain'
+overland_FlatICP.Mannings.Type = "Constant"
+overland_FlatICP.Mannings.GeomNames = "domain"
 overland_FlatICP.Mannings.Geom.domain.Value = 0.0003312
 
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland_FlatICP.PhaseSources.water.Type = 'Constant'
-overland_FlatICP.PhaseSources.water.GeomNames = 'domain'
+overland_FlatICP.PhaseSources.water.Type = "Constant"
+overland_FlatICP.PhaseSources.water.GeomNames = "domain"
 overland_FlatICP.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland_FlatICP.KnownSolution = 'NoKnownSolution'
+overland_FlatICP.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland_FlatICP.Solver = 'Richards'
+overland_FlatICP.Solver = "Richards"
 overland_FlatICP.Solver.MaxIter = 30000
 
 overland_FlatICP.Solver.Nonlinear.MaxIter = 300
 overland_FlatICP.Solver.Nonlinear.ResidualTol = 1e-8
-overland_FlatICP.Solver.Nonlinear.EtaChoice = 'Walker1'
-overland_FlatICP.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+overland_FlatICP.Solver.Nonlinear.EtaChoice = "Walker1"
+overland_FlatICP.Solver.Nonlinear.EtaChoice = "EtaConstant"
 overland_FlatICP.Solver.Nonlinear.EtaValue = 0.001
 overland_FlatICP.Solver.Nonlinear.UseJacobian = False
 overland_FlatICP.Solver.Nonlinear.DerivativeEpsilon = 1e-12
 overland_FlatICP.Solver.Nonlinear.StepTol = 1e-30
-overland_FlatICP.Solver.Nonlinear.Globalization = 'LineSearch'
+overland_FlatICP.Solver.Nonlinear.Globalization = "LineSearch"
 overland_FlatICP.Solver.Linear.KrylovDimension = 20
 overland_FlatICP.Solver.Linear.MaxRestart = 2
-overland_FlatICP.Solver.OverlandDiffusive.Epsilon = 1E-5
+overland_FlatICP.Solver.OverlandDiffusive.Epsilon = 1e-5
 
-overland_FlatICP.Solver.Linear.Preconditioner = 'PFMG'
+overland_FlatICP.Solver.Linear.Preconditioner = "PFMG"
 overland_FlatICP.Solver.PrintSubsurf = False
-overland_FlatICP.Solver.Drop = 1E-20
-overland_FlatICP.Solver.AbsTol = 1E-12
+overland_FlatICP.Solver.Drop = 1e-20
+overland_FlatICP.Solver.AbsTol = 1e-12
 
 overland_FlatICP.Solver.WriteSiloSubsurfData = False
 overland_FlatICP.Solver.WriteSiloPressure = False
 overland_FlatICP.Solver.WriteSiloSaturation = False
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
-overland_FlatICP.ICPressure.Type = 'HydroStaticPatch'
-overland_FlatICP.ICPressure.GeomNames = 'domain icsource'
+# ---------------------------------------------------------
+overland_FlatICP.ICPressure.Type = "HydroStaticPatch"
+overland_FlatICP.ICPressure.GeomNames = "domain icsource"
 
 overland_FlatICP.Geom.domain.ICPressure.Value = 0.0
-overland_FlatICP.Geom.domain.ICPressure.RefGeom = 'domain'
-overland_FlatICP.Geom.domain.ICPressure.RefPatch = 'z_upper'
+overland_FlatICP.Geom.domain.ICPressure.RefGeom = "domain"
+overland_FlatICP.Geom.domain.ICPressure.RefPatch = "z_upper"
 
 overland_FlatICP.Geom.icsource.ICPressure.Value = 0.1
-overland_FlatICP.Geom.icsource.ICPressure.RefGeom = 'domain'
-overland_FlatICP.Geom.icsource.ICPressure.RefPatch = 'z_upper'
+overland_FlatICP.Geom.icsource.ICPressure.RefGeom = "domain"
+overland_FlatICP.Geom.icsource.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 runcheck = 1
 
-#-----------------------------------------------------------------------------
-#original approach from K&M AWR 2006
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+# original approach from K&M AWR 2006
+# -----------------------------------------------------------------------------
 
-overland_FlatICP.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
+overland_FlatICP.Patch.z_upper.BCPressure.Type = "OverlandFlow"
 overland_FlatICP.Solver.Nonlinear.UseJacobian = False
-overland_FlatICP.Solver.Linear.Preconditioner.PCMatrixType = 'PFSymmetric'
+overland_FlatICP.Solver.Linear.Preconditioner.PCMatrixType = "PFSymmetric"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # New kinematic formulation - this should exactly match the original formulation
 #  for this flat test case
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland_FlatICP.Patch.z_upper.BCPressure.Type = 'OverlandKinematic'
+overland_FlatICP.Patch.z_upper.BCPressure.Type = "OverlandKinematic"
 overland_FlatICP.Solver.Nonlinear.UseJacobian = False
-overland_FlatICP.Solver.Linear.Preconditioner.PCMatrixType = 'PFSymmetric'
+overland_FlatICP.Solver.Linear.Preconditioner.PCMatrixType = "PFSymmetric"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Diffusive formulation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-#run with Jacobian False
-overland_FlatICP.Patch.z_upper.BCPressure.Type = 'OverlandDiffusive'
+# run with Jacobian False
+overland_FlatICP.Patch.z_upper.BCPressure.Type = "OverlandDiffusive"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Testing version compatibility
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-print('='*80)
-print('Test with version 1.0.0 of ParFlow')
-print('='*80)
+print("=" * 80)
+print("Test with version 1.0.0 of ParFlow")
+print("=" * 80)
 
-set_parflow_version('1.0.0')
+set_parflow_version("1.0.0")
 nb_error_v1 = overland_FlatICP.validate()
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-print('='*80)
-print('Test with version 3.6.0 of ParFlow')
-print('='*80)
+print("=" * 80)
+print("Test with version 3.6.0 of ParFlow")
+print("=" * 80)
 
-set_parflow_version('3.6.0')
+set_parflow_version("3.6.0")
 nb_error_v3 = overland_FlatICP.validate()
 
-print('='*80)
+print("=" * 80)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Asserts
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 found_error = False
 
 if nb_error_v1 != 1:
-  print(f'Expected to have 1 error with v1 but got {nb_error_v1}')
-  found_error = True
+    print(f"Expected to have 1 error with v1 but got {nb_error_v1}")
+    found_error = True
 
 if nb_error_v3 != 0:
-  print(f'Expected no errors with v3.6.0 but got {nb_error_v3}')
-  found_error = True
+    print(f"Expected no errors with v3.6.0 but got {nb_error_v3}")
+    found_error = True
 
 if found_error:
-  sys.exit(1)
-
+    sys.exit(1)
diff --git a/test/python/new_features/from_definition.py b/test/python/new_features/from_definition.py
index c3a2b6ede..0cd7e0402 100644
--- a/test/python/new_features/from_definition.py
+++ b/test/python/new_features/from_definition.py
@@ -1,6 +1,6 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Testing pfset with a .pfidb file
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 from pathlib import Path
 
@@ -9,8 +9,9 @@
 
 dsingle = Run("dsingle", __file__)
 
-dsingle.pfset(pfidb_file=get_absolute_path(
-    '$PF_SRC/test/correct_output/dsingle.pfidb.ref'))
+dsingle.pfset(
+    pfidb_file=get_absolute_path("$PF_SRC/test/correct_output/dsingle.pfidb.ref")
+)
 
 
 # Test pfidb
@@ -27,14 +28,14 @@
 
 
 # Test yaml
-generated, _ = dsingle.write(file_format='yaml')
+generated, _ = dsingle.write(file_format="yaml")
 old_text = Path(generated).read_text()
 
 # Make sure we are not repeating ourselves
 assert old_text != new_text
 
 dsingle2 = Run.from_definition(generated)
-generated, _ = dsingle2.write(file_format='yaml')
+generated, _ = dsingle2.write(file_format="yaml")
 
 new_text = Path(generated).read_text()
 
diff --git a/test/python/new_features/full_clone.py b/test/python/new_features/full_clone.py
index 17dba66dd..f783f3ac1 100644
--- a/test/python/new_features/full_clone.py
+++ b/test/python/new_features/full_clone.py
@@ -1,25 +1,29 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Testing Python clone function
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 import sys
 import os
 from parflow import Run
 from parflow.tools.fs import get_absolute_path
 
-test = Run('full_clone', __file__)
+test = Run("full_clone", __file__)
 
-test.pfset(yaml_file='$PF_SRC/test/correct_output/full_clone.yaml.ref', exit_if_undefined=True)
+test.pfset(
+    yaml_file="$PF_SRC/test/correct_output/full_clone.yaml.ref", exit_if_undefined=True
+)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 test.validate()
-generatedFile, runFile = test.write(file_format='yaml')
+generatedFile, runFile = test.write(file_format="yaml")
 
 # Prevent regression
-with open(generatedFile) as new, open(get_absolute_path('$PF_SRC/test/correct_output/full_clone.yaml.ref')) as ref:
-  if new.read() == ref.read():
-    print('Success we have the same file')
-  else:
-    print('Files are different')
-    sys.exit(1)
+with open(generatedFile) as new, open(
+    get_absolute_path("$PF_SRC/test/correct_output/full_clone.yaml.ref")
+) as ref:
+    if new.read() == ref.read():
+        print("Success we have the same file")
+    else:
+        print("Files are different")
+        sys.exit(1)
diff --git a/test/python/new_features/hyphen_test.py b/test/python/new_features/hyphen_test.py
index 94a806191..99744143c 100644
--- a/test/python/new_features/hyphen_test.py
+++ b/test/python/new_features/hyphen_test.py
@@ -1,9 +1,10 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Testing hyphenated user-defined tokens
 # (from default_overland)
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 from parflow import Run
+
 dover = Run("dover", __file__)
 
 dover.FileVersion = 4
@@ -12,9 +13,9 @@
 dover.Process.Topology.Q = 1
 dover.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 dover.ComputationalGrid.Lower.X = 0.0
 dover.ComputationalGrid.Lower.Y = 0.0
 dover.ComputationalGrid.Lower.Z = 0.0
@@ -25,26 +26,26 @@
 
 dover.ComputationalGrid.DX = 10.0
 dover.ComputationalGrid.DY = 10.0
-dover.ComputationalGrid.DZ = .05
+dover.ComputationalGrid.DZ = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
-dover.GeomInput.Names = 'domaininput leftinput rightinput channelinput'
+# ---------------------------------------------------------
+dover.GeomInput.Names = "domaininput leftinput rightinput channelinput"
 
-dover.GeomInput.domaininput.GeomName = 'domain'
-dover.GeomInput.leftinput.GeomName = 'left'
-dover.GeomInput.rightinput.GeomName = 'right'
-dover.GeomInput.channelinput.GeomName = 'channel'
+dover.GeomInput.domaininput.GeomName = "domain"
+dover.GeomInput.leftinput.GeomName = "left"
+dover.GeomInput.rightinput.GeomName = "right"
+dover.GeomInput.channelinput.GeomName = "channel"
 
-dover.GeomInput.domaininput.InputType = 'Box'
-dover.GeomInput.leftinput.InputType = 'Box'
-dover.GeomInput.rightinput.InputType = 'Box'
-dover.GeomInput.channelinput.InputType = 'Box'
+dover.GeomInput.domaininput.InputType = "Box"
+dover.GeomInput.leftinput.InputType = "Box"
+dover.GeomInput.rightinput.InputType = "Box"
+dover.GeomInput.channelinput.InputType = "Box"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 dover.Geom.domain.Lower.X = 0.0
 dover.Geom.domain.Lower.Y = 0.0
 dover.Geom.domain.Lower.Z = 0.0
@@ -52,11 +53,11 @@
 dover.Geom.domain.Upper.X = 300.0
 dover.Geom.domain.Upper.Y = 300.0
 dover.Geom.domain.Upper.Z = 1.5
-dover.Geom.domain.Patches = 'x-lower x-upper y-lower y-upper z-lower z-upper'
+dover.Geom.domain.Patches = "x-lower x-upper y-lower y-upper z-lower z-upper"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Left Slope Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 dover.Geom.left.Lower.X = 0.0
 dover.Geom.left.Lower.Y = 0.0
 dover.Geom.left.Lower.Z = 0.0
@@ -65,9 +66,9 @@
 dover.Geom.left.Upper.Y = 300.0
 dover.Geom.left.Upper.Z = 1.5
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Right Slope Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 dover.Geom.right.Lower.X = 160.0
 dover.Geom.right.Lower.Y = 0.0
 dover.Geom.right.Lower.Z = 0.0
@@ -76,9 +77,9 @@
 dover.Geom.right.Upper.Y = 300.0
 dover.Geom.right.Upper.Z = 1.5
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Channel Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 dover.Geom.channel.Lower.X = 140.0
 dover.Geom.channel.Lower.Y = 0.0
 dover.Geom.channel.Lower.Z = 0.0
@@ -87,11 +88,11 @@
 dover.Geom.channel.Upper.Y = 300.0
 dover.Geom.channel.Upper.Z = 1.5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Geom.Perm.Names = 'left right channel'
+dover.Geom.Perm.Names = "left right channel"
 
 # Values in m/hour
 
@@ -99,9 +100,9 @@
 # the following is ignored if the perm.type "Constant" settings are not
 # commented out, below.
 
-dover.Geom.left.Perm.Type = 'TurnBands'
-dover.Geom.left.Perm.LambdaX = 50.
-dover.Geom.left.Perm.LambdaY = 50.
+dover.Geom.left.Perm.Type = "TurnBands"
+dover.Geom.left.Perm.LambdaX = 50.0
+dover.Geom.left.Perm.LambdaY = 50.0
 dover.Geom.left.Perm.LambdaZ = 0.5
 dover.Geom.left.Perm.GeomMean = 0.01
 
@@ -111,13 +112,13 @@
 dover.Geom.left.Perm.KMax = 100.0
 dover.Geom.left.Perm.DelK = 0.2
 dover.Geom.left.Perm.Seed = 33333
-dover.Geom.left.Perm.LogNormal = 'Log'
-dover.Geom.left.Perm.StratType = 'Bottom'
+dover.Geom.left.Perm.LogNormal = "Log"
+dover.Geom.left.Perm.StratType = "Bottom"
 
 
-dover.Geom.right.Perm.Type = 'TurnBands'
-dover.Geom.right.Perm.LambdaX = 50.
-dover.Geom.right.Perm.LambdaY = 50.
+dover.Geom.right.Perm.Type = "TurnBands"
+dover.Geom.right.Perm.LambdaX = 50.0
+dover.Geom.right.Perm.LambdaY = 50.0
 dover.Geom.right.Perm.LambdaZ = 0.5
 dover.Geom.right.Perm.GeomMean = 0.05
 
@@ -127,8 +128,8 @@
 dover.Geom.right.Perm.KMax = 100.0
 dover.Geom.right.Perm.DelK = 0.2
 dover.Geom.right.Perm.Seed = 13333
-dover.Geom.right.Perm.LogNormal = 'Log'
-dover.Geom.right.Perm.StratType = 'Bottom'
+dover.Geom.right.Perm.LogNormal = "Log"
+dover.Geom.right.Perm.StratType = "Bottom"
 
 # hydraulic conductivity is very low, but not zero, top node will have to saturate
 # before overland flow can begin and will be driven by hortonian flow
@@ -137,64 +138,64 @@
 # subsurface
 #
 
-dover.Geom.left.Perm.Type = 'Constant'
+dover.Geom.left.Perm.Type = "Constant"
 dover.Geom.left.Perm.Value = 0.001
 
-dover.Geom.right.Perm.Type = 'Constant'
+dover.Geom.right.Perm.Type = "Constant"
 dover.Geom.right.Perm.Value = 0.01
 
-dover.Geom.channel.Perm.Type = 'Constant'
+dover.Geom.channel.Perm.Type = "Constant"
 dover.Geom.channel.Perm.Value = 0.00001
 
-dover.Perm.TensorType = 'TensorByGeom'
+dover.Perm.TensorType = "TensorByGeom"
 
-dover.Geom.Perm.TensorByGeom.Names = 'domain'
+dover.Geom.Perm.TensorByGeom.Names = "domain"
 
 dover.Geom.domain.Perm.TensorValX = 1.0
 dover.Geom.domain.Perm.TensorValY = 1.0
 dover.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.SpecificStorage.Type = 'Constant'
-dover.SpecificStorage.GeomNames = 'domain'
+dover.SpecificStorage.Type = "Constant"
+dover.SpecificStorage.GeomNames = "domain"
 dover.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Phase.Names = 'water'
+dover.Phase.Names = "water"
 
-dover.Phase.water.Density.Type = 'Constant'
+dover.Phase.water.Density.Type = "Constant"
 dover.Phase.water.Density.Value = 1.0
 
-dover.Phase.water.Viscosity.Type = 'Constant'
+dover.Phase.water.Viscosity.Type = "Constant"
 dover.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Contaminants.Names = ''
+dover.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Geom.Retardation.GeomNames = ''
+dover.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dover.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 #
 dover.TimingInfo.BaseUnit = 0.1
@@ -202,194 +203,194 @@
 dover.TimingInfo.StartTime = 0.0
 dover.TimingInfo.StopTime = 0.4
 dover.TimingInfo.DumpInterval = -1
-dover.TimeStep.Type = 'Constant'
+dover.TimeStep.Type = "Constant"
 dover.TimeStep.Value = 0.1
 #
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Geom.Porosity.GeomNames = 'left right channel'
+dover.Geom.Porosity.GeomNames = "left right channel"
 
-dover.Geom.left.Porosity.Type = 'Constant'
+dover.Geom.left.Porosity.Type = "Constant"
 dover.Geom.left.Porosity.Value = 0.25
 
-dover.Geom.right.Porosity.Type = 'Constant'
+dover.Geom.right.Porosity.Type = "Constant"
 dover.Geom.right.Porosity.Value = 0.25
 
-dover.Geom.channel.Porosity.Type = 'Constant'
+dover.Geom.channel.Porosity.Type = "Constant"
 dover.Geom.channel.Porosity.Value = 0.01
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Domain.GeomName = 'domain'
+dover.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Phase.RelPerm.Type = 'VanGenuchten'
-dover.Phase.RelPerm.GeomNames = 'domain'
+dover.Phase.RelPerm.Type = "VanGenuchten"
+dover.Phase.RelPerm.GeomNames = "domain"
 
 dover.Geom.domain.RelPerm.Alpha = 6.0
-dover.Geom.domain.RelPerm.N = 2.
+dover.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.Phase.Saturation.Type = 'VanGenuchten'
-dover.Phase.Saturation.GeomNames = 'domain'
+dover.Phase.Saturation.Type = "VanGenuchten"
+dover.Phase.Saturation.GeomNames = "domain"
 
 dover.Geom.domain.Saturation.Alpha = 6.0
-dover.Geom.domain.Saturation.N = 2.
+dover.Geom.domain.Saturation.N = 2.0
 dover.Geom.domain.Saturation.SRes = 0.2
 dover.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Wells.Names = ''
+dover.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Cycle.Names = 'constant rainrec'
-dover.Cycle.constant.Names = 'alltime'
+dover.Cycle.Names = "constant rainrec"
+dover.Cycle.constant.Names = "alltime"
 dover.Cycle.constant.alltime.Length = 1
 dover.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-dover.Cycle.rainrec.Names = 'rain rec'
+dover.Cycle.rainrec.Names = "rain rec"
 dover.Cycle.rainrec.rain.Length = 1
 dover.Cycle.rainrec.rec.Length = 2
 dover.Cycle.rainrec.Repeat = -1
 #
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 dover.BCPressure.PatchNames = dover.Geom.domain.Patches
 
-dover.Patch['x-lower'].BCPressure.Type = 'FluxConst'
-dover.Patch['x-lower'].BCPressure.Cycle = 'constant'
-dover.Patch['x-lower'].BCPressure.alltime.Value = 0.0
+dover.Patch["x-lower"].BCPressure.Type = "FluxConst"
+dover.Patch["x-lower"].BCPressure.Cycle = "constant"
+dover.Patch["x-lower"].BCPressure.alltime.Value = 0.0
 
-dover.Patch['y-lower'].BCPressure.Type = 'FluxConst'
-dover.Patch['y-lower'].BCPressure.Cycle = 'constant'
-dover.Patch['y-lower'].BCPressure.alltime.Value = 0.0
+dover.Patch["y-lower"].BCPressure.Type = "FluxConst"
+dover.Patch["y-lower"].BCPressure.Cycle = "constant"
+dover.Patch["y-lower"].BCPressure.alltime.Value = 0.0
 
-dover.Patch['z-lower'].BCPressure.Type = 'FluxConst'
-dover.Patch['z-lower'].BCPressure.Cycle = 'constant'
-dover.Patch['z-lower'].BCPressure.alltime.Value = 0.0
+dover.Patch["z-lower"].BCPressure.Type = "FluxConst"
+dover.Patch["z-lower"].BCPressure.Cycle = "constant"
+dover.Patch["z-lower"].BCPressure.alltime.Value = 0.0
 
-dover.Patch['x-upper'].BCPressure.Type = 'FluxConst'
-dover.Patch['x-upper'].BCPressure.Cycle = 'constant'
-dover.Patch['x-upper'].BCPressure.alltime.Value = 0.0
+dover.Patch["x-upper"].BCPressure.Type = "FluxConst"
+dover.Patch["x-upper"].BCPressure.Cycle = "constant"
+dover.Patch["x-upper"].BCPressure.alltime.Value = 0.0
 
-dover.Patch['y-upper'].BCPressure.Type = 'FluxConst'
-dover.Patch['y-upper'].BCPressure.Cycle = 'constant'
-dover.Patch['y-upper'].BCPressure.alltime.Value = 0.0
+dover.Patch["y-upper"].BCPressure.Type = "FluxConst"
+dover.Patch["y-upper"].BCPressure.Cycle = "constant"
+dover.Patch["y-upper"].BCPressure.alltime.Value = 0.0
 
 ## overland flow boundary condition with very heavy rainfall then slight ET
-dover.Patch['z-upper'].BCPressure.Type = 'OverlandFlow'
-dover.Patch['z-upper'].BCPressure.Cycle = 'rainrec'
-dover.Patch['z-upper'].BCPressure.rain.Value = -0.05
-dover.Patch['z-upper'].BCPressure.rec.Value = 0.000001
+dover.Patch["z-upper"].BCPressure.Type = "OverlandFlow"
+dover.Patch["z-upper"].BCPressure.Cycle = "rainrec"
+dover.Patch["z-upper"].BCPressure.rain.Value = -0.05
+dover.Patch["z-upper"].BCPressure.rec.Value = 0.000001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.TopoSlopesX.Type = 'Constant'
-dover.TopoSlopesX.GeomNames = 'left right channel'
+dover.TopoSlopesX.Type = "Constant"
+dover.TopoSlopesX.GeomNames = "left right channel"
 dover.TopoSlopesX.Geom.left.Value = -0.005
 dover.TopoSlopesX.Geom.right.Value = 0.005
 dover.TopoSlopesX.Geom.channel.Value = 0.00
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 
-dover.TopoSlopesY.Type = 'Constant'
-dover.TopoSlopesY.GeomNames = 'left right channel'
+dover.TopoSlopesY.Type = "Constant"
+dover.TopoSlopesY.GeomNames = "left right channel"
 dover.TopoSlopesY.Geom.left.Value = 0.001
 dover.TopoSlopesY.Geom.right.Value = 0.001
 dover.TopoSlopesY.Geom.channel.Value = 0.001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover.Mannings.Type = 'Constant'
-dover.Mannings.GeomNames = 'left right channel'
-dover.Mannings.Geom.left.Value = 5.e-6
-dover.Mannings.Geom.right.Value = 5.e-6
-dover.Mannings.Geom.channel.Value = 1.e-6
+dover.Mannings.Type = "Constant"
+dover.Mannings.GeomNames = "left right channel"
+dover.Mannings.Geom.left.Value = 5.0e-6
+dover.Mannings.Geom.right.Value = 5.0e-6
+dover.Mannings.Geom.channel.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.PhaseSources.water.Type = 'Constant'
-dover.PhaseSources.water.GeomNames = 'domain'
+dover.PhaseSources.water.Type = "Constant"
+dover.PhaseSources.water.GeomNames = "domain"
 dover.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.KnownSolution = 'NoKnownSolution'
+dover.KnownSolution = "NoKnownSolution"
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover.Solver = 'Richards'
+dover.Solver = "Richards"
 dover.Solver.MaxIter = 2500
 
 dover.Solver.Nonlinear.MaxIter = 20
 dover.Solver.Nonlinear.ResidualTol = 1e-9
-dover.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+dover.Solver.Nonlinear.EtaChoice = "EtaConstant"
 dover.Solver.Nonlinear.EtaValue = 0.01
 dover.Solver.Nonlinear.UseJacobian = False
 dover.Solver.Nonlinear.DerivativeEpsilon = 1e-8
 dover.Solver.Nonlinear.StepTol = 1e-20
-dover.Solver.Nonlinear.Globalization = 'LineSearch'
+dover.Solver.Nonlinear.Globalization = "LineSearch"
 dover.Solver.Linear.KrylovDimension = 20
 dover.Solver.Linear.MaxRestart = 2
 
-dover.Solver.Linear.Preconditioner = 'PFMGOctree'
+dover.Solver.Linear.Preconditioner = "PFMGOctree"
 dover.Solver.PrintSubsurf = False
-dover.Solver.Drop = 1E-20
-dover.Solver.AbsTol = 1E-9
+dover.Solver.Drop = 1e-20
+dover.Solver.AbsTol = 1e-9
 
 dover.Solver.WriteSiloSubsurfData = True
 dover.Solver.WriteSiloPressure = True
 dover.Solver.WriteSiloSaturation = True
 dover.Solver.WriteSiloConcentration = True
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be at the bottom of the domain, the top layer is initially dry
-dover.ICPressure.Type = 'HydroStaticPatch'
-dover.ICPressure.GeomNames = 'domain'
+dover.ICPressure.Type = "HydroStaticPatch"
+dover.ICPressure.GeomNames = "domain"
 dover.Geom.domain.ICPressure.Value = -3.0
 
-dover.Geom.domain.ICPressure.RefGeom = 'domain'
-dover.Geom.domain.ICPressure.RefPatch = 'z-upper'
+dover.Geom.domain.ICPressure.RefGeom = "domain"
+dover.Geom.domain.ICPressure.RefPatch = "z-upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dover.validate()
diff --git a/test/python/new_features/image-as-mask.py b/test/python/new_features/image-as-mask.py
index fca113c41..879426151 100644
--- a/test/python/new_features/image-as-mask.py
+++ b/test/python/new_features/image-as-mask.py
@@ -1,6 +1,6 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # example for pfsol generation from images
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 from parflow import Run
 from parflow.tools.fs import get_absolute_path
@@ -9,31 +9,34 @@
 
 sabino = Run("sabino", __file__)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # PFSOL generation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 color_to_patch = {
-  '#FFFFFF': 0, # Discard
-  '#466CFF': 2, # River
-  '#C56C00': 3, # Land
-  '#1900FF': 4, # Sea
-  '#3DFFFF': 5, # Lake
+    "#FFFFFF": 0,  # Discard
+    "#466CFF": 2,  # River
+    "#C56C00": 3,  # Land
+    "#1900FF": 4,  # Sea
+    "#3DFFFF": 5,  # Lake
 }
 
 mask = load_patch_matrix_from_image_file(
-  get_absolute_path('$PF_SRC/test/input/mask.png'))
+    get_absolute_path("$PF_SRC/test/input/mask.png")
+)
 
 top_patches = load_patch_matrix_from_image_file(
-  get_absolute_path('$PF_SRC/test/input/mask_top.png'), color_to_patch, fall_back_id=6)
+    get_absolute_path("$PF_SRC/test/input/mask_top.png"), color_to_patch, fall_back_id=6
+)
 
 side_patches = load_patch_matrix_from_image_file(
-  get_absolute_path('$PF_SRC/test/input/mask_side.png'), color_to_patch, fall_back_id=3)
+    get_absolute_path("$PF_SRC/test/input/mask_side.png"),
+    color_to_patch,
+    fall_back_id=3,
+)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-SolidFileBuilder(bottom=1) \
-  .mask(mask) \
-  .top_ids(top_patches) \
-  .side_ids(side_patches) \
-  .write('sabino_domain.pfsol', xllcorner=0, yllcorner=0, cellsize=90, vtk=True)
+SolidFileBuilder(bottom=1).mask(mask).top_ids(top_patches).side_ids(side_patches).write(
+    "sabino_domain.pfsol", xllcorner=0, yllcorner=0, cellsize=90, vtk=True
+)
diff --git a/test/python/new_features/normalize_location.py b/test/python/new_features/normalize_location.py
index 572c26d25..2cef61415 100644
--- a/test/python/new_features/normalize_location.py
+++ b/test/python/new_features/normalize_location.py
@@ -1,6 +1,6 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Testing pfset with a .pfidb file
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 from parflow.tools.helper import normalize_location
 
@@ -9,21 +9,21 @@
 def new_path(x):
     return x
 
-assert new_path('') == '.'
-assert new_path('.path') == 'path'
-assert new_path('path.to') == 'path/to'
-assert new_path('.path.to.the.obj') == 'path/to/the/obj'
-assert new_path('.path/to/the.obj') == 'path/to/the/obj'
-assert new_path('./path/to/the/obj') == 'path/to/the/obj'
-assert new_path('./path/to/the//obj/../../to/././the') == 'path/to/to/the'
+
+assert new_path("") == "."
+assert new_path(".path") == "path"
+assert new_path("path.to") == "path/to"
+assert new_path(".path.to.the.obj") == "path/to/the/obj"
+assert new_path(".path/to/the.obj") == "path/to/the/obj"
+assert new_path("./path/to/the/obj") == "path/to/the/obj"
+assert new_path("./path/to/the//obj/../../to/././the") == "path/to/to/the"
 
 # These should be no-op
 no_op_strings = [
-    '.',
-    '..'
-    'path',
-    '/path/to/the/obj',
-    'path/to/the/obj',
+    ".",
+    ".." "path",
+    "/path/to/the/obj",
+    "path/to/the/obj",
 ]
 
 for s in no_op_strings:
diff --git a/test/python/new_features/os_function.py b/test/python/new_features/os_function.py
index 54365a1d6..c89e25f66 100644
--- a/test/python/new_features/os_function.py
+++ b/test/python/new_features/os_function.py
@@ -1,6 +1,6 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Testing copy, remove, and mkdir methods on Python Run object
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 import sys
 
@@ -10,43 +10,46 @@
 # Update the working directory via the __file__ arg
 os_fxn = Run("os_fxn", __file__)
 
+
 def checkOK(file_path):
-  if not exists(file_path):
-    sys.exit(1)
+    if not exists(file_path):
+        sys.exit(1)
+
 
 def checkKO(file_path):
-  if exists(file_path):
-    sys.exit(1)
+    if exists(file_path):
+        sys.exit(1)
+
 
 # copying file from adjacent directory
-cp('$PF_SRC/test/input/BasicSettings.yaml')
-checkOK('BasicSettings.yaml')
+cp("$PF_SRC/test/input/BasicSettings.yaml")
+checkOK("BasicSettings.yaml")
 
-cp('$PF_SRC/test/input/BasicSettings.yaml', 'TestCopyFile.yaml')
-checkOK('TestCopyFile.yaml')
+cp("$PF_SRC/test/input/BasicSettings.yaml", "TestCopyFile.yaml")
+checkOK("TestCopyFile.yaml")
 
 # copying file from adjacent directory with environment variable
-cp('$PF_SRC/README.md')
-checkOK('README.md')
+cp("$PF_SRC/README.md")
+checkOK("README.md")
 
 # removing files
-rm('BasicSettings.yaml')
-checkKO('BasicSettings.yaml')
-rm('README.md')
-checkKO('README.md')
+rm("BasicSettings.yaml")
+checkKO("BasicSettings.yaml")
+rm("README.md")
+checkKO("README.md")
 
 # creating directory
-checkKO('test_directory')
-mkdir('test_directory')
-checkOK('test_directory')
+checkKO("test_directory")
+mkdir("test_directory")
+checkOK("test_directory")
 
 # creating same directory - you should get a message that this directory already exists.
-mkdir('test_directory')
+mkdir("test_directory")
 
-mkdir('test_directory/test1/test2/')
-checkOK('test_directory/test1/test2')
+mkdir("test_directory/test1/test2/")
+checkOK("test_directory/test1/test2")
 
 # removing directory
-rm('test_directory')
-rm('test_directory')
-checkKO('test_directory')
+rm("test_directory")
+rm("test_directory")
+checkKO("test_directory")
diff --git a/test/python/new_features/pfb_mask.py b/test/python/new_features/pfb_mask.py
index 6f77aedee..14feb8cb4 100644
--- a/test/python/new_features/pfb_mask.py
+++ b/test/python/new_features/pfb_mask.py
@@ -1,7 +1,7 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # example for pfsol generation
 # Testing pfb to mask generation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 from parflow import Run
 from parflow.tools.fs import get_absolute_path
@@ -10,17 +10,17 @@
 
 sabino = Run("sabino", __file__)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set Processor topology
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 sabino.Process.Topology.P = 1
 sabino.Process.Topology.Q = 1
 sabino.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 sabino.ComputationalGrid.Lower.X = 0.0
 sabino.ComputationalGrid.Lower.Y = 0.0
@@ -34,33 +34,34 @@
 sabino.ComputationalGrid.NY = 70
 sabino.ComputationalGrid.NZ = 20
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.GeomInput.Names = 'domaininput'
+sabino.GeomInput.Names = "domaininput"
 
-sabino.GeomInput.domaininput.GeomName = 'domain'
-sabino.GeomInput.domaininput.GeomNames = 'domain'
+sabino.GeomInput.domaininput.GeomName = "domain"
+sabino.GeomInput.domaininput.GeomNames = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # PFSOL generation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino_mask = load_patch_matrix_from_pfb_file(get_absolute_path('$PF_SRC/test/input/sabino_mask.pfb'))
+sabino_mask = load_patch_matrix_from_pfb_file(
+    get_absolute_path("$PF_SRC/test/input/sabino_mask.pfb")
+)
 
-SolidFileBuilder(top=1, bottom=2, side=3) \
-  .mask(sabino_mask) \
-  .write('sabino_domain.pfsol', xllcorner=0, yllcorner=0, cellsize=90, vtk=True) \
-  .for_key(sabino.GeomInput.domaininput)
+SolidFileBuilder(top=1, bottom=2, side=3).mask(sabino_mask).write(
+    "sabino_domain.pfsol", xllcorner=0, yllcorner=0, cellsize=90, vtk=True
+).for_key(sabino.GeomInput.domaininput)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Geom.domain.Patches = 'top bottom edge'
+sabino.Geom.domain.Patches = "top bottom edge"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 sabino.Geom.domain.Lower.X = 0.0
 sabino.Geom.domain.Lower.Y = 0.0
@@ -70,180 +71,180 @@
 sabino.Geom.domain.Upper.Y = 6300.0
 sabino.Geom.domain.Upper.Z = 2000.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Permeability (values in m/hr)
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Geom.Perm.Names = 'domain'
+sabino.Geom.Perm.Names = "domain"
 
-sabino.Geom.domain.Perm.Type = 'Constant'
+sabino.Geom.domain.Perm.Type = "Constant"
 sabino.Geom.domain.Perm.Value = 0.0018
 
-sabino.Perm.TensorType = 'TensorByGeom'
-sabino.Geom.Perm.TensorByGeom.Names = 'domain'
+sabino.Perm.TensorType = "TensorByGeom"
+sabino.Geom.Perm.TensorByGeom.Names = "domain"
 sabino.Geom.domain.Perm.TensorValX = 1.0
 sabino.Geom.domain.Perm.TensorValY = 1.0
 sabino.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.SpecificStorage.Type = 'Constant'
-sabino.SpecificStorage.GeomNames = 'domain'
+sabino.SpecificStorage.Type = "Constant"
+sabino.SpecificStorage.GeomNames = "domain"
 sabino.Geom.domain.SpecificStorage.Value = 1.0e-5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Phase.Names = 'water'
-sabino.Phase.water.Density.Type = 'Constant'
+sabino.Phase.Names = "water"
+sabino.Phase.water.Density.Type = "Constant"
 sabino.Phase.water.Density.Value = 1.0
-sabino.Phase.water.Viscosity.Type = 'Constant'
+sabino.Phase.water.Viscosity.Type = "Constant"
 sabino.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Contaminants.Names = ''
+sabino.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 sabino.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Timing (time units is set by units of permeability)
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 sabino.TimingInfo.BaseUnit = 1.0
 sabino.TimingInfo.StartCount = 0
 sabino.TimingInfo.StartTime = 0.0
 sabino.TimingInfo.StopTime = 6.0
 sabino.TimingInfo.DumpInterval = 1.0
-sabino.TimeStep.Type = 'Constant'
+sabino.TimeStep.Type = "Constant"
 sabino.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Geom.Porosity.GeomNames = 'domain'
-sabino.Geom.domain.Porosity.Type = 'Constant'
+sabino.Geom.Porosity.GeomNames = "domain"
+sabino.Geom.domain.Porosity.Type = "Constant"
 sabino.Geom.domain.Porosity.Value = 0.1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Domain.GeomName = 'domain'
+sabino.Domain.GeomName = "domain"
 
-#----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 # Mobility
-#----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 
-sabino.Phase.water.Mobility.Type = 'Constant'
+sabino.Phase.water.Mobility.Type = "Constant"
 sabino.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Wells.Names = ''
+sabino.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Cycle.Names = 'constant'
-sabino.Cycle.constant.Names = 'alltime'
+sabino.Cycle.Names = "constant"
+sabino.Cycle.constant.Names = "alltime"
 sabino.Cycle.constant.alltime.Length = 1
 sabino.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.BCPressure.PatchNames = 'edge top bottom'
+sabino.BCPressure.PatchNames = "edge top bottom"
 
-sabino.Patch.edge.BCPressure.Type = 'FluxConst'
-sabino.Patch.edge.BCPressure.Cycle = 'constant'
+sabino.Patch.edge.BCPressure.Type = "FluxConst"
+sabino.Patch.edge.BCPressure.Cycle = "constant"
 sabino.Patch.edge.BCPressure.alltime.Value = 0.0
 
-sabino.Patch.bottom.BCPressure.Type = 'FluxConst'
-sabino.Patch.bottom.BCPressure.Cycle = 'constant'
+sabino.Patch.bottom.BCPressure.Type = "FluxConst"
+sabino.Patch.bottom.BCPressure.Cycle = "constant"
 sabino.Patch.bottom.BCPressure.alltime.Value = 0.0
 
 sabino.Patch.top.BCPressure.alltime.Value = -0.0035
-sabino.Patch.top.BCPressure.Type = 'OverlandFlow'
-sabino.Patch.top.BCPressure.Cycle = 'constant'
+sabino.Patch.top.BCPressure.Type = "OverlandFlow"
+sabino.Patch.top.BCPressure.Cycle = "constant"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Topo slopes in x-direction
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.TopoSlopesX.Type = 'Constant'
-sabino.TopoSlopesX.GeomNames = 'domain'
+sabino.TopoSlopesX.Type = "Constant"
+sabino.TopoSlopesX.GeomNames = "domain"
 sabino.TopoSlopesX.Geom.domain.Value = -0.005
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Topo slopes in y-direction
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.TopoSlopesY.Type = 'Constant'
-sabino.TopoSlopesY.GeomNames = 'domain'
+sabino.TopoSlopesY.Type = "Constant"
+sabino.TopoSlopesY.GeomNames = "domain"
 sabino.TopoSlopesY.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mannings coefficient
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Mannings.Type = 'Constant'
-sabino.Mannings.GeomNames = 'domain'
+sabino.Mannings.Type = "Constant"
+sabino.Mannings.GeomNames = "domain"
 sabino.Mannings.Geom.domain.Value = 5.52e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Phase.RelPerm.Type = 'VanGenuchten'
-sabino.Phase.RelPerm.GeomNames = 'domain'
+sabino.Phase.RelPerm.Type = "VanGenuchten"
+sabino.Phase.RelPerm.GeomNames = "domain"
 sabino.Geom.domain.RelPerm.Alpha = 3.5
 sabino.Geom.domain.RelPerm.N = 2.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Saturation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Phase.Saturation.Type = 'VanGenuchten'
-sabino.Phase.Saturation.GeomNames = 'domain'
+sabino.Phase.Saturation.Type = "VanGenuchten"
+sabino.Phase.Saturation.GeomNames = "domain"
 
 sabino.Geom.domain.Saturation.Alpha = 3.5
-sabino.Geom.domain.Saturation.N = 2.
+sabino.Geom.domain.Saturation.N = 2.0
 sabino.Geom.domain.Saturation.SRes = 0.1
 sabino.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.PhaseSources.water.Type = 'Constant'
-sabino.PhaseSources.water.GeomNames = 'domain'
+sabino.PhaseSources.water.Type = "Constant"
+sabino.PhaseSources.water.GeomNames = "domain"
 sabino.PhaseSources.water.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-sabino.ICPressure.Type = 'Constant'
-sabino.ICPressure.GeomNames = 'domain'
-sabino.Geom.domain.ICPressure.RefPatch = 'top'
+sabino.ICPressure.Type = "Constant"
+sabino.ICPressure.GeomNames = "domain"
+sabino.Geom.domain.ICPressure.RefPatch = "top"
 sabino.Geom.domain.ICPressure.Value = -10
 
-#----------------------------------------------------------------
+# ----------------------------------------------------------------
 # Outputs
 # ------------------------------------------------------------
 
@@ -252,39 +253,39 @@
 sabino.Solver.PrintSaturation = True
 sabino.Solver.PrintMask = True
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.KnownSolution = 'NoKnownSolution'
+sabino.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Solver = 'Richards'
+sabino.Solver = "Richards"
 sabino.Solver.TerrainFollowingGrid = True
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setting up vertical layering
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 sabino.Solver.Nonlinear.VariableDz = True
-sabino.dzScale.GeomNames = 'domain'
-sabino.dzScale.Type = 'nzList'
+sabino.dzScale.GeomNames = "domain"
+sabino.dzScale.Type = "nzList"
 sabino.dzScale.nzListNumber = 20
 sabino.Cell._0.dzScale.Value = 2.0
 sabino.Cell._1.dzScale.Value = 2.0
 sabino.Cell._2.dzScale.Value = 2.0
 sabino.Cell._3.dzScale.Value = 1.0
 sabino.Cell._4.dzScale.Value = 1.0
-sabino.Cell._5.dzScale.Value = .50
-sabino.Cell._6.dzScale.Value = .50
-sabino.Cell._7.dzScale.Value = .20
-sabino.Cell._8.dzScale.Value = .20
-sabino.Cell._9.dzScale.Value = .20
-sabino.Cell._10.dzScale.Value = .20
-sabino.Cell._11.dzScale.Value = .10
+sabino.Cell._5.dzScale.Value = 0.50
+sabino.Cell._6.dzScale.Value = 0.50
+sabino.Cell._7.dzScale.Value = 0.20
+sabino.Cell._8.dzScale.Value = 0.20
+sabino.Cell._9.dzScale.Value = 0.20
+sabino.Cell._10.dzScale.Value = 0.20
+sabino.Cell._11.dzScale.Value = 0.10
 sabino.Cell._12.dzScale.Value = 0.02
 sabino.Cell._13.dzScale.Value = 0.02
 sabino.Cell._14.dzScale.Value = 0.02
@@ -295,26 +296,26 @@
 sabino.Cell._19.dzScale.Value = 0.001
 
 sabino.Solver.MaxIter = 25000000
-sabino.Solver.Drop = 1E-20
-sabino.Solver.AbsTol = 1E-8
+sabino.Solver.Drop = 1e-20
+sabino.Solver.AbsTol = 1e-8
 sabino.Solver.MaxConvergenceFailures = 8
 sabino.Solver.Nonlinear.MaxIter = 80
 sabino.Solver.Nonlinear.ResidualTol = 1e-6
 
-sabino.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+sabino.Solver.Nonlinear.EtaChoice = "EtaConstant"
 sabino.Solver.Nonlinear.EtaValue = 0.001
 sabino.Solver.Nonlinear.UseJacobian = True
 sabino.Solver.Nonlinear.DerivativeEpsilon = 1e-16
 sabino.Solver.Nonlinear.StepTol = 1e-30
-sabino.Solver.Nonlinear.Globalization = 'LineSearch'
+sabino.Solver.Nonlinear.Globalization = "LineSearch"
 sabino.Solver.Linear.KrylovDimension = 70
 sabino.Solver.Linear.MaxRestarts = 2
 
-sabino.Solver.Linear.Preconditioner = 'PFMG'
-sabino.Solver.Linear.Preconditioner.PCMatrixType = 'FullJacobian'
+sabino.Solver.Linear.Preconditioner = "PFMG"
+sabino.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Distribute inputs and run simulation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.run()
\ No newline at end of file
+sabino.run()
diff --git a/test/python/new_features/pfb_reader.py b/test/python/new_features/pfb_reader.py
new file mode 100644
index 000000000..b945a1f94
--- /dev/null
+++ b/test/python/new_features/pfb_reader.py
@@ -0,0 +1,27 @@
+import os
+import sys
+import numpy as np
+from parflow import Run
+from parflow.tools.io import read_pfb, write_pfb, ParflowBinaryReader
+from parflow.tools.fs import mkdir, get_absolute_path
+
+# Check that the ParflowBinaryReader reads the correct values of p, q, r even
+# when they don't exactly divide nx, ny, nz.
+
+data = np.ones((89, 91))
+run = Run("test_dist", __file__)
+working_dir = get_absolute_path(os.path.join("test_output", "dist"))
+mkdir(working_dir)
+
+path = os.path.join(working_dir, "data.pfb")
+write_pfb(path, data)
+
+run.dist(path, P=15, Q=15)
+rd = ParflowBinaryReader(path, read_sg_info=True)
+if rd.header["p"] != 15 or rd.header["q"] != 15:
+    sys.exit(1)
+
+run.dist(path, P=14, Q=15)
+rd = ParflowBinaryReader(path, read_sg_info=True)
+if rd.header["p"] != 14 or rd.header["q"] != 15:
+    sys.exit(1)
diff --git a/test/python/new_features/pfidb_pfset.py b/test/python/new_features/pfidb_pfset.py
index d13fd757c..72e0250b3 100644
--- a/test/python/new_features/pfidb_pfset.py
+++ b/test/python/new_features/pfidb_pfset.py
@@ -1,6 +1,6 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Testing pfset with a .pfidb file
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 import sys
 from parflow import Run
@@ -8,20 +8,23 @@
 
 dsingle = Run("dsingle", __file__)
 
-dsingle.pfset(pfidb_file=get_absolute_path('$PF_SRC/test/correct_output/dsingle.pfidb.ref'))
+dsingle.pfset(
+    pfidb_file=get_absolute_path("$PF_SRC/test/correct_output/dsingle.pfidb.ref")
+)
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Write and compare the ParFlow database files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 generatedFile, runArg = dsingle.write()
 
 # Prevent regression
-with open(generatedFile) as new, \
-    open(get_absolute_path('$PF_SRC/test/correct_output/dsingle.pfidb.ref')) as ref:
-  if new.read() == ref.read():
-    print('Success we have the same file')
-  else:
-    print('Files are different')
-    sys.exit(1)
+with open(generatedFile) as new, open(
+    get_absolute_path("$PF_SRC/test/correct_output/dsingle.pfidb.ref")
+) as ref:
+    if new.read() == ref.read():
+        print("Success we have the same file")
+    else:
+        print("Files are different")
+        sys.exit(1)
diff --git a/test/python/new_features/pfset_test.py b/test/python/new_features/pfset_test.py
index 779ce75b3..956df3097 100644
--- a/test/python/new_features/pfset_test.py
+++ b/test/python/new_features/pfset_test.py
@@ -1,99 +1,104 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 #  Testing pfset python function for setting keys that aren't in the library
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 import sys
 from parflow import Run
 from parflow.tools.fs import get_absolute_path
 
-pfset_test = Run('pfset_test', __file__)
+pfset_test = Run("pfset_test", __file__)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
-pfset_test.Domain.GeomName = 'domain'
+# -----------------------------------------------------------------------------
+pfset_test.Domain.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
-pfset_test.GeomInput.Names = "domain_input background_input source_region_input concen_region_input"
+# ---------------------------------------------------------
+pfset_test.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
-pfset_test.GeomInput.domain_input.InputType = 'Box'
-pfset_test.GeomInput.domain_input.GeomName = 'domain'
+# ---------------------------------------------------------
+pfset_test.GeomInput.domain_input.InputType = "Box"
+pfset_test.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry Input
-#---------------------------------------------------------
-pfset_test.GeomInput.background_input.InputType = 'Box'
-pfset_test.GeomInput.background_input.GeomName = 'background'
+# ---------------------------------------------------------
+pfset_test.GeomInput.background_input.InputType = "Box"
+pfset_test.GeomInput.background_input.GeomName = "background"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry Input
-#---------------------------------------------------------
-pfset_test.GeomInput.source_region_input.InputType = 'Box'
-pfset_test.GeomInput.source_region_input.GeomName = 'source_region'
+# ---------------------------------------------------------
+pfset_test.GeomInput.source_region_input.InputType = "Box"
+pfset_test.GeomInput.source_region_input.GeomName = "source_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry Input
-#---------------------------------------------------------
-pfset_test.GeomInput.concen_region_input.InputType = 'Box'
-pfset_test.GeomInput.concen_region_input.GeomName = 'concen_region'
+# ---------------------------------------------------------
+pfset_test.GeomInput.concen_region_input.InputType = "Box"
+pfset_test.GeomInput.concen_region_input.GeomName = "concen_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Phase names
-#---------------------------------------------------------
-pfset_test.Phase.Names = 'water'
+# ---------------------------------------------------------
+pfset_test.Phase.Names = "water"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # pfset: Key/Value
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # Test key that does not exist
-pfset_test.pfset(key='A.New.Key.Test', value='SomeSuperContent')
+pfset_test.pfset(key="A.New.Key.Test", value="SomeSuperContent")
 # Test key that does not exist with partial valid path
-pfset_test.pfset(key='Process.Topology.Random.Path', value=5)
+pfset_test.pfset(key="Process.Topology.Random.Path", value=5)
 # Test key that does not exist from a child element
-pfset_test.Process.Topology.pfset(key='Random.PathFromTopology', value=6)
+pfset_test.Process.Topology.pfset(key="Random.PathFromTopology", value=6)
 # Test setting a valid value from a full path
-pfset_test.pfset(key='Process.Topology.P', value=2)
+pfset_test.pfset(key="Process.Topology.P", value=2)
 # Test setting a valid value from a relative path
-pfset_test.Process.pfset(key='Topology.Q', value=3)
+pfset_test.Process.pfset(key="Topology.Q", value=3)
 # Test setting a valid field
-pfset_test.Process.Topology.pfset(key='R', value=4)
+pfset_test.Process.Topology.pfset(key="R", value=4)
 # Test setting an invalid field
-pfset_test.Process.Topology.pfset(key='Seb', value=5)
+pfset_test.Process.Topology.pfset(key="Seb", value=5)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # pfset: hierarchical_map
-#-----------------------------------------------------------------------------
-
-pfset_test.pfset(hierarchical_map={
-    'SpecificStorage': {
-        'Type': 'Constant',
-        'GeomNames': 'domain',
+# -----------------------------------------------------------------------------
+
+pfset_test.pfset(
+    hierarchical_map={
+        "SpecificStorage": {
+            "Type": "Constant",
+            "GeomNames": "domain",
+        }
     }
-})
+)
 
-constOne = {'Type': 'Constant', 'Value': 1.0}
+constOne = {"Type": "Constant", "Value": 1.0}
 pfset_test.Phase.water.Density.pfset(hierarchical_map=constOne)
 pfset_test.Phase.water.Viscosity.pfset(flat_map=constOne)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # pfset: yaml_file
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfset_test.pfset(yaml_file='$PF_SRC/test/input/BasicSettings.yaml')
-pfset_test.pfset(yaml_file='$PF_SRC/test/input/ComputationalGrid.yaml')
-pfset_test.Geom.pfset(yaml_file='$PF_SRC/test/input/GeomChildren.yaml')
+pfset_test.pfset(yaml_file="$PF_SRC/test/input/BasicSettings.yaml")
+pfset_test.pfset(yaml_file="$PF_SRC/test/input/ComputationalGrid.yaml")
+pfset_test.Geom.pfset(yaml_file="$PF_SRC/test/input/GeomChildren.yaml")
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # pfset: yaml_content
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfset_test.Geom.source_region.pfset(yaml_content='''
+pfset_test.Geom.source_region.pfset(
+    yaml_content="""
 Lower:
   X: 65.56
   Y: 79.34
@@ -102,10 +107,12 @@
   X: 74.44
   Y: 89.99
   Z: 5.5
-''')
+"""
+)
 
 
-pfset_test.Geom.concen_region.pfset(yaml_content='''
+pfset_test.Geom.concen_region.pfset(
+    yaml_content="""
 Lower:
   X: 60.0
   Y: 80.0
@@ -114,32 +121,39 @@
   X: 80.0
   Y: 100.0
   Z: 6.0
-''')
+"""
+)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # pfset: flat_map
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfset_test.pfset(flat_map={
-    'Phase.Saturation.Type': 'VanGenuchten',
-    'Phase.Saturation.GeomNames': 'domain',
-})
+pfset_test.pfset(
+    flat_map={
+        "Phase.Saturation.Type": "VanGenuchten",
+        "Phase.Saturation.GeomNames": "domain",
+    }
+)
 
-pfset_test.Phase.pfset(flat_map={
-    'RelPerm.Type': 'VanGenuchten',
-    'RelPerm.GeomNames': 'domain',
-})
+pfset_test.Phase.pfset(
+    flat_map={
+        "RelPerm.Type": "VanGenuchten",
+        "RelPerm.GeomNames": "domain",
+    }
+)
 
-#-----------------------------------------------------------------------------
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 pfset_test.validate()
-generatedFile, runFile = pfset_test.write(file_format='yaml')
+generatedFile, runFile = pfset_test.write(file_format="yaml")
 
 # Prevent regression
-with open(generatedFile) as new, open(get_absolute_path('$PF_SRC/test/correct_output/pfset_test.yaml.ref')) as ref:
-  if new.read() == ref.read():
-    print('Success we have the same file')
-  else:
-    print('Files are different')
-    sys.exit(1)
+with open(generatedFile) as new, open(
+    get_absolute_path("$PF_SRC/test/correct_output/pfset_test.yaml.ref")
+) as ref:
+    if new.read() == ref.read():
+        print("Success we have the same file")
+    else:
+        print("Files are different")
+        sys.exit(1)
diff --git a/test/python/new_features/prefix_naming.py b/test/python/new_features/prefix_naming.py
index fd181c38f..c741e1d0d 100644
--- a/test/python/new_features/prefix_naming.py
+++ b/test/python/new_features/prefix_naming.py
@@ -1,10 +1,11 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 #  This is an adaptation of the LW_var_dz.py test
 #  This test demonstrates the different ways to set integers within a key name
 #  See dzScale and Coeff keys
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 from parflow import Run
+
 prefix = Run("prefix", __file__)
 
 prefix.FileVersion = 4
@@ -13,9 +14,9 @@
 prefix.Process.Topology.Q = 1
 prefix.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 prefix.ComputationalGrid.Lower.X = 0.0
 prefix.ComputationalGrid.Lower.Y = 0.0
 prefix.ComputationalGrid.Lower.Z = 0.0
@@ -28,21 +29,21 @@
 
 prefix.ComputationalGrid.DX = 1000.0
 prefix.ComputationalGrid.DY = 1000.0
-#"native" grid resolution is 2m everywhere X NZ=25 for 50m
-#computational domain.
+# "native" grid resolution is 2m everywhere X NZ=25 for 50m
+# computational domain.
 prefix.ComputationalGrid.DZ = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
-prefix.GeomInput.Names = 'domaininput'
+# ---------------------------------------------------------
+prefix.GeomInput.Names = "domaininput"
 
-prefix.GeomInput.domaininput.GeomName = 'domain'
-prefix.GeomInput.domaininput.InputType = 'Box'
+prefix.GeomInput.domaininput.GeomName = "domain"
+prefix.GeomInput.domaininput.InputType = "Box"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 prefix.Geom.domain.Lower.X = 0.0
 prefix.Geom.domain.Lower.Y = 0.0
 prefix.Geom.domain.Lower.Z = 0.0
@@ -51,21 +52,21 @@
 prefix.Geom.domain.Upper.Y = 32000.0
 # this upper is synched to computational grid, not linked w/ Z multipliers
 prefix.Geom.domain.Upper.Z = 12.0
-prefix.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+prefix.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#--------------------------------------------
+# --------------------------------------------
 # variable dz assignments
-#--------------------------------------------
+# --------------------------------------------
 prefix.Solver.Nonlinear.VariableDz = True
-prefix.dzScale.GeomNames = 'domain'
-prefix.dzScale.Type = 'nzList'
+prefix.dzScale.GeomNames = "domain"
+prefix.dzScale.Type = "nzList"
 prefix.dzScale.nzListNumber = 6
 
 # Here are four different ways to set integer values as part of a key name:
 # 1) bracket, quotes, underscore
-prefix.Cell['_0'].dzScale.Value = 1.0
+prefix.Cell["_0"].dzScale.Value = 1.0
 # 2) bracket, quotes, no underscore
-prefix.Cell['1'].dzScale.Value = 1.00
+prefix.Cell["1"].dzScale.Value = 1.00
 # 3) bracket, no quotes, no underscore
 prefix.Cell[2].dzScale.Value = 1.000
 # 4) no bracket, no quotes, underscore
@@ -73,18 +74,18 @@
 prefix.Cell._4.dzScale.Value = 1.000
 prefix.Cell._5.dzScale.Value = 0.05
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-prefix.Geom.Perm.Names = 'domain'
+prefix.Geom.Perm.Names = "domain"
 
 # Values in m/hour
 
 
-prefix.Geom.domain.Perm.Type = 'Constant'
+prefix.Geom.domain.Perm.Type = "Constant"
 
-prefix.Geom.domain.Perm.Type = 'TurnBands'
+prefix.Geom.domain.Perm.Type = "TurnBands"
 prefix.Geom.domain.Perm.LambdaX = 5000.0
 prefix.Geom.domain.Perm.LambdaY = 5000.0
 prefix.Geom.domain.Perm.LambdaZ = 50.0
@@ -92,151 +93,150 @@
 
 prefix.Geom.domain.Perm.Sigma = 0.20
 prefix.Geom.domain.Perm.Sigma = 1.20
-#pfset Geom.domain.Perm.Sigma   0.48989794
+# pfset Geom.domain.Perm.Sigma   0.48989794
 prefix.Geom.domain.Perm.NumLines = 150
 prefix.Geom.domain.Perm.RZeta = 10.0
 prefix.Geom.domain.Perm.KMax = 100.0000001
 prefix.Geom.domain.Perm.DelK = 0.2
 prefix.Geom.domain.Perm.Seed = 33333
-prefix.Geom.domain.Perm.LogNormal = 'Log'
-prefix.Geom.domain.Perm.StratType = 'Bottom'
+prefix.Geom.domain.Perm.LogNormal = "Log"
+prefix.Geom.domain.Perm.StratType = "Bottom"
 
 
-prefix.Perm.TensorType = 'TensorByGeom'
+prefix.Perm.TensorType = "TensorByGeom"
 
-prefix.Geom.Perm.TensorByGeom.Names = 'domain'
+prefix.Geom.Perm.TensorByGeom.Names = "domain"
 
 prefix.Geom.domain.Perm.TensorValX = 1.0
 prefix.Geom.domain.Perm.TensorValY = 1.0
 prefix.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-prefix.SpecificStorage.Type = 'Constant'
-prefix.SpecificStorage.GeomNames = 'domain'
+prefix.SpecificStorage.Type = "Constant"
+prefix.SpecificStorage.GeomNames = "domain"
 prefix.Geom.domain.SpecificStorage.Value = 1.0e-5
-#pfset Geom.domain.SpecificStorage.Value 0.0
+# pfset Geom.domain.SpecificStorage.Value 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-prefix.Phase.Names = 'water'
+prefix.Phase.Names = "water"
 
-prefix.Phase.water.Density.Type = 'Constant'
+prefix.Phase.water.Density.Type = "Constant"
 prefix.Phase.water.Density.Value = 1.0
 
-prefix.Phase.water.Viscosity.Type = 'Constant'
+prefix.Phase.water.Viscosity.Type = "Constant"
 prefix.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-prefix.Contaminants.Names = ''
+prefix.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-prefix.Geom.Retardation.GeomNames = ''
+prefix.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 prefix.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 prefix.TimingInfo.BaseUnit = 10.0
 prefix.TimingInfo.StartCount = 0
 prefix.TimingInfo.StartTime = 0.0
 prefix.TimingInfo.StopTime = 200.0
 prefix.TimingInfo.DumpInterval = 20.0
-prefix.TimeStep.Type = 'Constant'
+prefix.TimeStep.Type = "Constant"
 prefix.TimeStep.Value = 10.0
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-prefix.Geom.Porosity.GeomNames = 'domain'
+prefix.Geom.Porosity.GeomNames = "domain"
 
-prefix.Geom.domain.Porosity.Type = 'Constant'
+prefix.Geom.domain.Porosity.Type = "Constant"
 prefix.Geom.domain.Porosity.Value = 0.25
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-prefix.Domain.GeomName = 'domain'
+prefix.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-prefix.Phase.RelPerm.Type = 'VanGenuchten'
-prefix.Phase.RelPerm.GeomNames = 'domain'
+prefix.Phase.RelPerm.Type = "VanGenuchten"
+prefix.Phase.RelPerm.GeomNames = "domain"
 
 prefix.Geom.domain.RelPerm.Alpha = 1.0
-prefix.Geom.domain.RelPerm.N = 3.
+prefix.Geom.domain.RelPerm.N = 3.0
 
 # Another example
 prefix.Geom.domain.RelPerm.Coeff._0 = 0.5
-prefix.Geom.domain.RelPerm.Coeff['_1'] = 1.0
-prefix.Geom.domain.RelPerm.Coeff['2'] = 1.0
+prefix.Geom.domain.RelPerm.Coeff["_1"] = 1.0
+prefix.Geom.domain.RelPerm.Coeff["2"] = 1.0
 prefix.Geom.domain.RelPerm.Coeff[3] = 1.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-prefix.Phase.Saturation.Type = 'VanGenuchten'
-prefix.Phase.Saturation.GeomNames = 'domain'
+prefix.Phase.Saturation.Type = "VanGenuchten"
+prefix.Phase.Saturation.GeomNames = "domain"
 
 prefix.Geom.domain.Saturation.Alpha = 1.0
 prefix.Geom.domain.Saturation.Alpha = 1.0
-prefix.Geom.domain.Saturation.N = 3.
+prefix.Geom.domain.Saturation.N = 3.0
 prefix.Geom.domain.Saturation.SRes = 0.1
 prefix.Geom.domain.Saturation.SSat = 1.0
 
 
-
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
-prefix.Wells.Names = ''
+# -----------------------------------------------------------------------------
+prefix.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
-prefix.Cycle.Names = 'constant rainrec'
-prefix.Cycle.constant.Names = 'alltime'
+# -----------------------------------------------------------------------------
+prefix.Cycle.Names = "constant rainrec"
+prefix.Cycle.constant.Names = "alltime"
 prefix.Cycle.constant.alltime.Length = 10000000
 prefix.Cycle.constant.Repeat = -1
 
-prefix.Cycle.rainrec.Names = 'rain rec'
+prefix.Cycle.rainrec.Names = "rain rec"
 prefix.Cycle.rainrec.rain.Length = 10
 prefix.Cycle.rainrec.rec.Length = 20
 prefix.Cycle.rainrec.Repeat = 14
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 prefix.BCPressure.PatchNames = prefix.Geom.domain.Patches
 
-prefix.Patch.x_lower.BCPressure.Type = 'DirEquilPLinear'
-prefix.Patch.x_lower.BCPressure.Cycle = 'constant'
+prefix.Patch.x_lower.BCPressure.Type = "DirEquilPLinear"
+prefix.Patch.x_lower.BCPressure.Cycle = "constant"
 prefix.Patch.x_lower.BCPressure.alltime.NumPoints = 2
 
 # Another example
 prefix.Patch.x_lower.BCPressure.alltime._0.Location = 0.0
-prefix.Patch.x_lower.BCPressure.alltime['0'].Value = 0.0
-prefix.Patch.x_lower.BCPressure.alltime['1'].Location = 0.0
+prefix.Patch.x_lower.BCPressure.alltime["0"].Value = 0.0
+prefix.Patch.x_lower.BCPressure.alltime["1"].Location = 0.0
 prefix.Patch.x_lower.BCPressure.alltime[1].Value = 14.0
 prefix.Patch.x_lower.BCPressure.alltime.XLower = 0.0
 prefix.Patch.x_lower.BCPressure.alltime.YLower = 0.0
@@ -244,70 +244,70 @@
 prefix.Patch.x_lower.BCPressure.alltime.YUpper = 1.0
 
 
-prefix.Patch.y_lower.BCPressure.Type = 'FluxConst'
-prefix.Patch.y_lower.BCPressure.Cycle = 'constant'
+prefix.Patch.y_lower.BCPressure.Type = "FluxConst"
+prefix.Patch.y_lower.BCPressure.Cycle = "constant"
 prefix.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-prefix.Patch.z_lower.BCPressure.Type = 'FluxConst'
-prefix.Patch.z_lower.BCPressure.Cycle = 'constant'
+prefix.Patch.z_lower.BCPressure.Type = "FluxConst"
+prefix.Patch.z_lower.BCPressure.Cycle = "constant"
 prefix.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-prefix.Patch.x_upper.BCPressure.Type = 'FluxConst'
-prefix.Patch.x_upper.BCPressure.Cycle = 'constant'
+prefix.Patch.x_upper.BCPressure.Type = "FluxConst"
+prefix.Patch.x_upper.BCPressure.Cycle = "constant"
 prefix.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-prefix.Patch.y_upper.BCPressure.Type = 'FluxConst'
-prefix.Patch.y_upper.BCPressure.Cycle = 'constant'
+prefix.Patch.y_upper.BCPressure.Type = "FluxConst"
+prefix.Patch.y_upper.BCPressure.Cycle = "constant"
 prefix.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-prefix.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-prefix.Patch.z_upper.BCPressure.Cycle = 'constant'
+prefix.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+prefix.Patch.z_upper.BCPressure.Cycle = "constant"
 prefix.Patch.z_upper.BCPressure.alltime.Value = -0.005
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-prefix.TopoSlopesX.Type = 'Constant'
-prefix.TopoSlopesX.GeomNames = 'domain'
+prefix.TopoSlopesX.Type = "Constant"
+prefix.TopoSlopesX.GeomNames = "domain"
 prefix.TopoSlopesX.Geom.domain.Value = 0.05
 
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-prefix.TopoSlopesY.Type = 'Constant'
-prefix.TopoSlopesY.GeomNames = 'domain'
+prefix.TopoSlopesY.Type = "Constant"
+prefix.TopoSlopesY.GeomNames = "domain"
 prefix.TopoSlopesY.Geom.domain.Value = -0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-prefix.Mannings.Type = 'Constant'
-prefix.Mannings.GeomNames = 'domain'
+prefix.Mannings.Type = "Constant"
+prefix.Mannings.GeomNames = "domain"
 prefix.Mannings.Geom.domain.Value = 0.00005
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-prefix.PhaseSources.water.Type = 'Constant'
-prefix.PhaseSources.water.GeomNames = 'domain'
+prefix.PhaseSources.water.Type = "Constant"
+prefix.PhaseSources.water.GeomNames = "domain"
 prefix.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-prefix.KnownSolution = 'NoKnownSolution'
+prefix.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-prefix.Solver = 'Richards'
+prefix.Solver = "Richards"
 prefix.Solver.MaxIter = 2500
 
 prefix.Solver.TerrainFollowingGrid = True
@@ -319,39 +319,39 @@
 
 
 prefix.Solver.PrintSubsurf = False
-prefix.Solver.Drop = 1E-20
-prefix.Solver.AbsTol = 1E-10
+prefix.Solver.Drop = 1e-20
+prefix.Solver.AbsTol = 1e-10
 
 
-prefix.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+prefix.Solver.Nonlinear.EtaChoice = "EtaConstant"
 prefix.Solver.Nonlinear.EtaValue = 0.001
 prefix.Solver.Nonlinear.UseJacobian = True
 prefix.Solver.Nonlinear.DerivativeEpsilon = 1e-14
 prefix.Solver.Nonlinear.StepTol = 1e-25
-prefix.Solver.Nonlinear.Globalization = 'LineSearch'
+prefix.Solver.Nonlinear.Globalization = "LineSearch"
 prefix.Solver.Linear.KrylovDimension = 80
 prefix.Solver.Linear.MaxRestarts = 2
 
-prefix.Solver.Linear.Preconditioner = 'MGSemi'
-prefix.Solver.Linear.Preconditioner = 'PFMG'
-prefix.Solver.Linear.Preconditioner.PCMatrixType = 'FullJacobian'
+prefix.Solver.Linear.Preconditioner = "MGSemi"
+prefix.Solver.Linear.Preconditioner = "PFMG"
+prefix.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be at the bottom of the domain, the top layer is initially dry
-prefix.ICPressure.Type = 'HydroStaticPatch'
-prefix.ICPressure.GeomNames = 'domain'
+prefix.ICPressure.Type = "HydroStaticPatch"
+prefix.ICPressure.GeomNames = "domain"
 prefix.Geom.domain.ICPressure.Value = -10.0
 
-prefix.Geom.domain.ICPressure.RefGeom = 'domain'
-prefix.Geom.domain.ICPressure.RefPatch = 'z_upper'
+prefix.Geom.domain.ICPressure.RefGeom = "domain"
+prefix.Geom.domain.ICPressure.RefPatch = "z_upper"
 
 prefix.Solver.Spinup = False
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 prefix.run()
diff --git a/test/python/new_features/selection_methods.py b/test/python/new_features/selection_methods.py
index b3818e96f..fea74b4db 100644
--- a/test/python/new_features/selection_methods.py
+++ b/test/python/new_features/selection_methods.py
@@ -1,97 +1,112 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Testing various methods that use locations for selections
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 from parflow import Run
 from parflow.tools.fs import get_absolute_path
 
 run = Run("run", __file__)
 
-run.pfset(pfidb_file=get_absolute_path(
-    '$PF_SRC/test/correct_output/dsingle.pfidb.ref'))
+run.pfset(pfidb_file=get_absolute_path("$PF_SRC/test/correct_output/dsingle.pfidb.ref"))
 
 # Some modifications for testing
-run.Solver = 'Richards'
+run.Solver = "Richards"
 
 # select() tests
 porosity = run.Geom.Porosity
-assert run.Geom.Porosity.select('.')[0] is porosity
-assert porosity.select('.')[0] is porosity
+assert run.Geom.Porosity.select(".")[0] is porosity
+assert porosity.select(".")[0] is porosity
 assert porosity.select()[0] is porosity
-assert porosity.select('..')[0] is run.Geom
-assert porosity.select('GeomNames') == porosity.GeomNames
-assert porosity.select('./GeomNames') == porosity.GeomNames
-assert run.Geom.select('Porosity/GeomNames') == porosity.GeomNames
-assert porosity.select('/')[0] is run
-assert porosity.select('/Geom')[0] is run.Geom
-assert porosity.select('/Geom/Porosity')[0] is porosity
-assert porosity.select('../../Solver/../Geom/./Porosity/..')[0] is run.Geom
-assert porosity.select('../..')[0] is run
-assert run.Geom.select('.Porosity.GeomNames') == porosity.GeomNames
+assert porosity.select("..")[0] is run.Geom
+assert porosity.select("GeomNames") == porosity.GeomNames
+assert porosity.select("./GeomNames") == porosity.GeomNames
+assert run.Geom.select("Porosity/GeomNames") == porosity.GeomNames
+assert porosity.select("/")[0] is run
+assert porosity.select("/Geom")[0] is run.Geom
+assert porosity.select("/Geom/Porosity")[0] is porosity
+assert porosity.select("../../Solver/../Geom/./Porosity/..")[0] is run.Geom
+assert porosity.select("../..")[0] is run
+assert run.Geom.select(".Porosity.GeomNames") == porosity.GeomNames
 
 # value() tests
 background = run.Geom.background
-assert (run.value('Geom/background/Porosity/Value') ==
-        run.Geom.background.Porosity.Value)
-assert (run.value('/Geom/background/Porosity/Value') ==
-        run.Geom.background.Porosity.Value)
-assert (background.value('/Geom/background/Porosity/Value')
-        == run.Geom.background.Porosity.Value)
-assert (background.value('tce/Retardation/Type') ==
-        run.Geom.background.tce.Retardation.Type)
-assert (background.value('./tce/Retardation/Type') ==
-        run.Geom.background.tce.Retardation.Type)
-assert (background.value('tce/Retardation/Rate') ==
-        run.Geom.background.tce.Retardation.Rate)
-assert (background.value('../concen_region/Lower/Z') ==
-        run.Geom.concen_region.Lower.Z)
-assert background.value('../domain/Patches') == run.Geom.domain.Patches
-assert run.Solver.value('.') == 'Richards'
-assert run.Solver.value() == 'Richards'
-assert run.Solver.CLM.value('..') == 'Richards'
-assert run.value('./UseClustering') == run.UseClustering
-assert run.value('./UseClustering') is not None
-assert run.value('./UseClustering', skip_default=True) is None
-assert run.value('./Geom/background/../.././Solver') == 'Richards'
-assert (run.value('Geom.background.Porosity.Value') ==
-        run.Geom.background.Porosity.Value)
-assert (run.value('Cell/_0/dzScale/Value') == run.Cell._0.dzScale.Value)
-assert (run.value('./Cell/_0/dzScale/Value') == run.Cell._0.dzScale.Value)
-assert (run.Cell.value('_0/dzScale/Value') == run.Cell._0.dzScale.Value)
-assert (run.Cell._0.dzScale.value('Value') == run.Cell._0.dzScale.Value)
+assert run.value("Geom/background/Porosity/Value") == run.Geom.background.Porosity.Value
+assert (
+    run.value("/Geom/background/Porosity/Value") == run.Geom.background.Porosity.Value
+)
+assert (
+    background.value("/Geom/background/Porosity/Value")
+    == run.Geom.background.Porosity.Value
+)
+assert (
+    background.value("tce/Retardation/Type") == run.Geom.background.tce.Retardation.Type
+)
+assert (
+    background.value("./tce/Retardation/Type")
+    == run.Geom.background.tce.Retardation.Type
+)
+assert (
+    background.value("tce/Retardation/Rate") == run.Geom.background.tce.Retardation.Rate
+)
+assert background.value("../concen_region/Lower/Z") == run.Geom.concen_region.Lower.Z
+assert background.value("../domain/Patches") == run.Geom.domain.Patches
+assert run.Solver.value(".") == "Richards"
+assert run.Solver.value() == "Richards"
+assert run.Solver.CLM.value("..") == "Richards"
+assert run.value("./UseClustering") == run.UseClustering
+assert run.value("./UseClustering") is not None
+assert run.value("./UseClustering", skip_default=True) is None
+assert run.value("./Geom/background/../.././Solver") == "Richards"
+assert run.value("Geom.background.Porosity.Value") == run.Geom.background.Porosity.Value
+assert run.value("Cell/_0/dzScale/Value") == run.Cell._0.dzScale.Value
+assert run.value("./Cell/_0/dzScale/Value") == run.Cell._0.dzScale.Value
+assert run.Cell.value("_0/dzScale/Value") == run.Cell._0.dzScale.Value
+assert run.Cell._0.dzScale.value("Value") == run.Cell._0.dzScale.Value
 
 # details() tests
 lower = run.Geom.background.Lower
 upper = run.Geom.background.Upper
-assert lower.details('X') is lower['_details_']['X']
-assert lower.details('./X') is lower['_details_']['X']
-assert lower.details('../Upper/Y') is upper['_details_']['Y']
-assert run.Solver.details('.') is run.Solver['_details_']['_value_']
-assert (run.Solver.details('.').get('default') ==
-        run.Solver['_details_']['_value_']['default'])
-assert run.Solver.CLM.details('..') is run.Solver['_details_']['_value_']
-assert (run.Solver.CLM.details('..').get('default') ==
-        run.Solver['_details_']['_value_']['default'])
-assert (run.details('Solver/TerrainFollowingGrid').get('default') ==
-        run.Solver.TerrainFollowingGrid['_details_']['_value_']['default'])
-assert (run.details('/Solver/TerrainFollowingGrid').get('default') ==
-        run.Solver.TerrainFollowingGrid['_details_']['_value_']['default'])
-assert (run.details('Solver.TerrainFollowingGrid').get('default') ==
-        run.Solver.TerrainFollowingGrid['_details_']['_value_']['default'])
+assert lower.details("X") is lower["_details_"]["X"]
+assert lower.details("./X") is lower["_details_"]["X"]
+assert lower.details("../Upper/Y") is upper["_details_"]["Y"]
+assert run.Solver.details(".") is run.Solver["_details_"]["_value_"]
+assert (
+    run.Solver.details(".").get("default")
+    == run.Solver["_details_"]["_value_"]["default"]
+)
+assert run.Solver.CLM.details("..") is run.Solver["_details_"]["_value_"]
+assert (
+    run.Solver.CLM.details("..").get("default")
+    == run.Solver["_details_"]["_value_"]["default"]
+)
+assert (
+    run.details("Solver/TerrainFollowingGrid").get("default")
+    == run.Solver.TerrainFollowingGrid["_details_"]["_value_"]["default"]
+)
+assert (
+    run.details("/Solver/TerrainFollowingGrid").get("default")
+    == run.Solver.TerrainFollowingGrid["_details_"]["_value_"]["default"]
+)
+assert (
+    run.details("Solver.TerrainFollowingGrid").get("default")
+    == run.Solver.TerrainFollowingGrid["_details_"]["_value_"]["default"]
+)
 
 # doc() tests
-assert run.doc('/Domain') == run.Domain.__doc__ + '\n'
-assert run.doc('Domain') == run.Domain.__doc__ + '\n'
-assert run.doc('./Domain') == run.Domain.__doc__ + '\n'
-assert run.Domain.doc('.') == run.Domain.__doc__ + '\n'
-assert run.Domain.doc() == run.Domain.__doc__ + '\n'
-assert (run.Domain.doc('../Solver') == run.Solver.__doc__ + '\n' +
-        run.Solver._details_['_value_']['help'] + '\n')
-assert background.doc('.') == background.__doc__ + '\n'
-assert (background.doc('./Lower/X') ==
-        background.Lower._details_['X']['help'] + '\n')
-assert (background.doc('./Lower/../..//./background/Upper/Y/.') ==
-        background.Upper._details_['Y']['help'] + '\n')
-assert background.Upper.doc('../..') == run.Geom.__doc__ + '\n'
-assert (background.doc('.Lower.X') ==
-        background.Lower._details_['X']['help'] + '\n')
+assert run.doc("/Domain") == run.Domain.__doc__ + "\n"
+assert run.doc("Domain") == run.Domain.__doc__ + "\n"
+assert run.doc("./Domain") == run.Domain.__doc__ + "\n"
+assert run.Domain.doc(".") == run.Domain.__doc__ + "\n"
+assert run.Domain.doc() == run.Domain.__doc__ + "\n"
+assert (
+    run.Domain.doc("../Solver")
+    == run.Solver.__doc__ + "\n" + run.Solver._details_["_value_"]["help"] + "\n"
+)
+assert background.doc(".") == background.__doc__ + "\n"
+assert background.doc("./Lower/X") == background.Lower._details_["X"]["help"] + "\n"
+assert (
+    background.doc("./Lower/../..//./background/Upper/Y/.")
+    == background.Upper._details_["Y"]["help"] + "\n"
+)
+assert background.Upper.doc("../..") == run.Geom.__doc__ + "\n"
+assert background.doc(".Lower.X") == background.Lower._details_["X"]["help"] + "\n"
diff --git a/test/python/new_features/serial_runs.py b/test/python/new_features/serial_runs.py
index ea4eb61e0..21cd52c97 100644
--- a/test/python/new_features/serial_runs.py
+++ b/test/python/new_features/serial_runs.py
@@ -1,8 +1,9 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 #  Testing serial runs within same script
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 from parflow import Run
+
 dover_1 = Run("dover_1", __file__)
 
 dover_1.FileVersion = 4
@@ -11,9 +12,9 @@
 dover_1.Process.Topology.Q = 1
 dover_1.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 dover_1.ComputationalGrid.Lower.X = 0.0
 dover_1.ComputationalGrid.Lower.Y = 0.0
 dover_1.ComputationalGrid.Lower.Z = 0.0
@@ -24,73 +25,73 @@
 
 dover_1.ComputationalGrid.DX = 10.0
 dover_1.ComputationalGrid.DY = 10.0
-dover_1.ComputationalGrid.DZ = .05
+dover_1.ComputationalGrid.DZ = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
-dover_1.GeomInput.Names = 'domaininput leftinput rightinput channelinput'
-
-dover_1.GeomInput.domaininput.GeomName = 'domain'
-dover_1.GeomInput.leftinput.GeomName = 'left'
-dover_1.GeomInput.rightinput.GeomName = 'right'
-dover_1.GeomInput.channelinput.GeomName = 'channel'
-
-dover_1.GeomInput.domaininput.InputType = 'Box'
-dover_1.GeomInput.leftinput.InputType = 'Box'
-dover_1.GeomInput.rightinput.InputType = 'Box'
-dover_1.GeomInput.channelinput.InputType = 'Box'
-
-#---------------------------------------------------------
-# Domain Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+dover_1.GeomInput.Names = "domaininput leftinput rightinput channelinput"
+
+dover_1.GeomInput.domaininput.GeomName = "domain"
+dover_1.GeomInput.leftinput.GeomName = "left"
+dover_1.GeomInput.rightinput.GeomName = "right"
+dover_1.GeomInput.channelinput.GeomName = "channel"
+
+dover_1.GeomInput.domaininput.InputType = "Box"
+dover_1.GeomInput.leftinput.InputType = "Box"
+dover_1.GeomInput.rightinput.InputType = "Box"
+dover_1.GeomInput.channelinput.InputType = "Box"
+
+# ---------------------------------------------------------
+# Domain Geometry
+# ---------------------------------------------------------
 dover_1.Geom.domain.Lower.X = 0.0
 dover_1.Geom.domain.Lower.Y = 0.0
 dover_1.Geom.domain.Lower.Z = 0.0
-#  
+#
 dover_1.Geom.domain.Upper.X = 300.0
 dover_1.Geom.domain.Upper.Y = 300.0
 dover_1.Geom.domain.Upper.Z = 1.5
-dover_1.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+dover_1.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#---------------------------------------------------------
-# Left Slope Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Left Slope Geometry
+# ---------------------------------------------------------
 dover_1.Geom.left.Lower.X = 0.0
 dover_1.Geom.left.Lower.Y = 0.0
 dover_1.Geom.left.Lower.Z = 0.0
-#  
+#
 dover_1.Geom.left.Upper.X = 140.0
 dover_1.Geom.left.Upper.Y = 300.0
 dover_1.Geom.left.Upper.Z = 1.5
 
-#---------------------------------------------------------
-# Right Slope Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Right Slope Geometry
+# ---------------------------------------------------------
 dover_1.Geom.right.Lower.X = 160.0
 dover_1.Geom.right.Lower.Y = 0.0
 dover_1.Geom.right.Lower.Z = 0.0
-#  
+#
 dover_1.Geom.right.Upper.X = 300.0
 dover_1.Geom.right.Upper.Y = 300.0
 dover_1.Geom.right.Upper.Z = 1.5
 
-#---------------------------------------------------------
-# Channel Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Channel Geometry
+# ---------------------------------------------------------
 dover_1.Geom.channel.Lower.X = 140.0
 dover_1.Geom.channel.Lower.Y = 0.0
 dover_1.Geom.channel.Lower.Z = 0.0
-#  
+#
 dover_1.Geom.channel.Upper.X = 160.0
 dover_1.Geom.channel.Upper.Y = 300.0
 dover_1.Geom.channel.Upper.Z = 1.5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover_1.Geom.Perm.Names = 'left right channel'
+dover_1.Geom.Perm.Names = "left right channel"
 
 # Values in m/hour
 
@@ -98,9 +99,9 @@
 # the following is ignored if the perm.type "Constant" settings are not
 # commented out, below.
 
-dover_1.Geom.left.Perm.Type = 'TurnBands'
-dover_1.Geom.left.Perm.LambdaX = 50.
-dover_1.Geom.left.Perm.LambdaY = 50.
+dover_1.Geom.left.Perm.Type = "TurnBands"
+dover_1.Geom.left.Perm.LambdaX = 50.0
+dover_1.Geom.left.Perm.LambdaY = 50.0
 dover_1.Geom.left.Perm.LambdaZ = 0.5
 dover_1.Geom.left.Perm.GeomMean = 0.01
 
@@ -110,13 +111,13 @@
 dover_1.Geom.left.Perm.KMax = 100.0
 dover_1.Geom.left.Perm.DelK = 0.2
 dover_1.Geom.left.Perm.Seed = 33333
-dover_1.Geom.left.Perm.LogNormal = 'Log'
-dover_1.Geom.left.Perm.StratType = 'Bottom'
+dover_1.Geom.left.Perm.LogNormal = "Log"
+dover_1.Geom.left.Perm.StratType = "Bottom"
 
 
-dover_1.Geom.right.Perm.Type = 'TurnBands'
-dover_1.Geom.right.Perm.LambdaX = 50.
-dover_1.Geom.right.Perm.LambdaY = 50.
+dover_1.Geom.right.Perm.Type = "TurnBands"
+dover_1.Geom.right.Perm.LambdaX = 50.0
+dover_1.Geom.right.Perm.LambdaY = 50.0
 dover_1.Geom.right.Perm.LambdaZ = 0.5
 dover_1.Geom.right.Perm.GeomMean = 0.05
 
@@ -126,8 +127,8 @@
 dover_1.Geom.right.Perm.KMax = 100.0
 dover_1.Geom.right.Perm.DelK = 0.2
 dover_1.Geom.right.Perm.Seed = 13333
-dover_1.Geom.right.Perm.LogNormal = 'Log'
-dover_1.Geom.right.Perm.StratType = 'Bottom'
+dover_1.Geom.right.Perm.LogNormal = "Log"
+dover_1.Geom.right.Perm.StratType = "Bottom"
 
 # hydraulic conductivity is very low, but not zero, top node will have to saturate
 # before overland flow can begin and will be driven by hortonian flow
@@ -136,270 +137,268 @@
 # subsurface
 #
 
-dover_1.Geom.left.Perm.Type = 'Constant'
+dover_1.Geom.left.Perm.Type = "Constant"
 dover_1.Geom.left.Perm.Value = 0.001
 
-dover_1.Geom.right.Perm.Type = 'Constant'
+dover_1.Geom.right.Perm.Type = "Constant"
 dover_1.Geom.right.Perm.Value = 0.01
 
-dover_1.Geom.channel.Perm.Type = 'Constant'
+dover_1.Geom.channel.Perm.Type = "Constant"
 dover_1.Geom.channel.Perm.Value = 0.00001
 
-dover_1.Perm.TensorType = 'TensorByGeom'
+dover_1.Perm.TensorType = "TensorByGeom"
 
-dover_1.Geom.Perm.TensorByGeom.Names = 'domain'
+dover_1.Geom.Perm.TensorByGeom.Names = "domain"
 
 dover_1.Geom.domain.Perm.TensorValX = 1.0
 dover_1.Geom.domain.Perm.TensorValY = 1.0
 dover_1.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover_1.SpecificStorage.Type = 'Constant'
-dover_1.SpecificStorage.GeomNames = 'domain'
+dover_1.SpecificStorage.Type = "Constant"
+dover_1.SpecificStorage.GeomNames = "domain"
 dover_1.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover_1.Phase.Names = 'water'
+dover_1.Phase.Names = "water"
 
-dover_1.Phase.water.Density.Type = 'Constant'
+dover_1.Phase.water.Density.Type = "Constant"
 dover_1.Phase.water.Density.Value = 1.0
 
-dover_1.Phase.water.Viscosity.Type = 'Constant'
+dover_1.Phase.water.Viscosity.Type = "Constant"
 dover_1.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover_1.Contaminants.Names = ''
+dover_1.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover_1.Geom.Retardation.GeomNames = ''
+dover_1.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dover_1.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-# 
+#
 dover_1.TimingInfo.BaseUnit = 0.1
 dover_1.TimingInfo.StartCount = 0
 dover_1.TimingInfo.StartTime = 0.0
 dover_1.TimingInfo.StopTime = 0.4
 dover_1.TimingInfo.DumpInterval = 0
-dover_1.TimeStep.Type = 'Constant'
+dover_1.TimeStep.Type = "Constant"
 dover_1.TimeStep.Value = 0.1
-#  
-#-----------------------------------------------------------------------------
+#
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover_1.Geom.Porosity.GeomNames = 'left right channel'
+dover_1.Geom.Porosity.GeomNames = "left right channel"
 
-dover_1.Geom.left.Porosity.Type = 'Constant'
+dover_1.Geom.left.Porosity.Type = "Constant"
 dover_1.Geom.left.Porosity.Value = 0.25
 
-dover_1.Geom.right.Porosity.Type = 'Constant'
+dover_1.Geom.right.Porosity.Type = "Constant"
 dover_1.Geom.right.Porosity.Value = 0.25
 
-dover_1.Geom.channel.Porosity.Type = 'Constant'
+dover_1.Geom.channel.Porosity.Type = "Constant"
 dover_1.Geom.channel.Porosity.Value = 0.01
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover_1.Domain.GeomName = 'domain'
+dover_1.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover_1.Phase.RelPerm.Type = 'VanGenuchten'
-dover_1.Phase.RelPerm.GeomNames = 'domain'
+dover_1.Phase.RelPerm.Type = "VanGenuchten"
+dover_1.Phase.RelPerm.GeomNames = "domain"
 
 dover_1.Geom.domain.RelPerm.Alpha = 6.0
-dover_1.Geom.domain.RelPerm.N = 2.
+dover_1.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover_1.Phase.Saturation.Type = 'VanGenuchten'
-dover_1.Phase.Saturation.GeomNames = 'domain'
+dover_1.Phase.Saturation.Type = "VanGenuchten"
+dover_1.Phase.Saturation.GeomNames = "domain"
 
 dover_1.Geom.domain.Saturation.Alpha = 6.0
-dover_1.Geom.domain.Saturation.N = 2.
+dover_1.Geom.domain.Saturation.N = 2.0
 dover_1.Geom.domain.Saturation.SRes = 0.2
 dover_1.Geom.domain.Saturation.SSat = 1.0
 
 
-
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
-dover_1.Wells.Names = ''
+# -----------------------------------------------------------------------------
+dover_1.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
-dover_1.Cycle.Names = 'constant rainrec'
-dover_1.Cycle.constant.Names = 'alltime'
+# -----------------------------------------------------------------------------
+dover_1.Cycle.Names = "constant rainrec"
+dover_1.Cycle.constant.Names = "alltime"
 dover_1.Cycle.constant.alltime.Length = 1
 dover_1.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-dover_1.Cycle.rainrec.Names = 'rain rec'
+dover_1.Cycle.rainrec.Names = "rain rec"
 dover_1.Cycle.rainrec.rain.Length = 1
 dover_1.Cycle.rainrec.rec.Length = 2
 dover_1.Cycle.rainrec.Repeat = -1
-#  
-#-----------------------------------------------------------------------------
+#
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 dover_1.BCPressure.PatchNames = dover_1.Geom.domain.Patches
 
-dover_1.Patch.x_lower.BCPressure.Type = 'FluxConst'
-dover_1.Patch.x_lower.BCPressure.Cycle = 'constant'
+dover_1.Patch.x_lower.BCPressure.Type = "FluxConst"
+dover_1.Patch.x_lower.BCPressure.Cycle = "constant"
 dover_1.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-dover_1.Patch.y_lower.BCPressure.Type = 'FluxConst'
-dover_1.Patch.y_lower.BCPressure.Cycle = 'constant'
+dover_1.Patch.y_lower.BCPressure.Type = "FluxConst"
+dover_1.Patch.y_lower.BCPressure.Cycle = "constant"
 dover_1.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-dover_1.Patch.z_lower.BCPressure.Type = 'FluxConst'
-dover_1.Patch.z_lower.BCPressure.Cycle = 'constant'
+dover_1.Patch.z_lower.BCPressure.Type = "FluxConst"
+dover_1.Patch.z_lower.BCPressure.Cycle = "constant"
 dover_1.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-dover_1.Patch.x_upper.BCPressure.Type = 'FluxConst'
-dover_1.Patch.x_upper.BCPressure.Cycle = 'constant'
+dover_1.Patch.x_upper.BCPressure.Type = "FluxConst"
+dover_1.Patch.x_upper.BCPressure.Cycle = "constant"
 dover_1.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-dover_1.Patch.y_upper.BCPressure.Type = 'FluxConst'
-dover_1.Patch.y_upper.BCPressure.Cycle = 'constant'
+dover_1.Patch.y_upper.BCPressure.Type = "FluxConst"
+dover_1.Patch.y_upper.BCPressure.Cycle = "constant"
 dover_1.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
 ## overland flow boundary condition with very heavy rainfall then slight ET
-dover_1.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-dover_1.Patch.z_upper.BCPressure.Cycle = 'rainrec'
+dover_1.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+dover_1.Patch.z_upper.BCPressure.Cycle = "rainrec"
 dover_1.Patch.z_upper.BCPressure.rain.Value = -0.05
 dover_1.Patch.z_upper.BCPressure.rec.Value = 0.000001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dover_1.TopoSlopesX.Type = 'Constant'
-dover_1.TopoSlopesX.GeomNames = 'left right channel'
+dover_1.TopoSlopesX.Type = "Constant"
+dover_1.TopoSlopesX.GeomNames = "left right channel"
 dover_1.TopoSlopesX.Geom.left.Value = -0.005
 dover_1.TopoSlopesX.Geom.right.Value = 0.005
 dover_1.TopoSlopesX.Geom.channel.Value = 0.00
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 
-dover_1.TopoSlopesY.Type = 'Constant'
-dover_1.TopoSlopesY.GeomNames = 'left right channel'
+dover_1.TopoSlopesY.Type = "Constant"
+dover_1.TopoSlopesY.GeomNames = "left right channel"
 dover_1.TopoSlopesY.Geom.left.Value = 0.001
 dover_1.TopoSlopesY.Geom.right.Value = 0.001
 dover_1.TopoSlopesY.Geom.channel.Value = 0.001
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-dover_1.Mannings.Type = 'Constant'
-dover_1.Mannings.GeomNames = 'left right channel'
-dover_1.Mannings.Geom.left.Value = 5.e-6
-dover_1.Mannings.Geom.right.Value = 5.e-6
-dover_1.Mannings.Geom.channel.Value = 1.e-6
+dover_1.Mannings.Type = "Constant"
+dover_1.Mannings.GeomNames = "left right channel"
+dover_1.Mannings.Geom.left.Value = 5.0e-6
+dover_1.Mannings.Geom.right.Value = 5.0e-6
+dover_1.Mannings.Geom.channel.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover_1.PhaseSources.water.Type = 'Constant'
-dover_1.PhaseSources.water.GeomNames = 'domain'
+dover_1.PhaseSources.water.Type = "Constant"
+dover_1.PhaseSources.water.GeomNames = "domain"
 dover_1.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover_1.KnownSolution = 'NoKnownSolution'
+dover_1.KnownSolution = "NoKnownSolution"
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dover_1.Solver = 'Richards'
+dover_1.Solver = "Richards"
 dover_1.Solver.MaxIter = 2500
 
 dover_1.Solver.Nonlinear.MaxIter = 20
 dover_1.Solver.Nonlinear.ResidualTol = 1e-9
-dover_1.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+dover_1.Solver.Nonlinear.EtaChoice = "EtaConstant"
 dover_1.Solver.Nonlinear.EtaValue = 0.01
 dover_1.Solver.Nonlinear.UseJacobian = False
 dover_1.Solver.Nonlinear.DerivativeEpsilon = 1e-8
 dover_1.Solver.Nonlinear.StepTol = 1e-20
-dover_1.Solver.Nonlinear.Globalization = 'LineSearch'
+dover_1.Solver.Nonlinear.Globalization = "LineSearch"
 dover_1.Solver.Linear.KrylovDimension = 20
 dover_1.Solver.Linear.MaxRestart = 2
 
-dover_1.Solver.Linear.Preconditioner = 'PFMGOctree'
+dover_1.Solver.Linear.Preconditioner = "PFMGOctree"
 dover_1.Solver.PrintSubsurf = False
-dover_1.Solver.Drop = 1E-20
-dover_1.Solver.AbsTol = 1E-9
+dover_1.Solver.Drop = 1e-20
+dover_1.Solver.AbsTol = 1e-9
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be at the bottom of the domain, the top layer is initially dry
-dover_1.ICPressure.Type = 'HydroStaticPatch'
-dover_1.ICPressure.GeomNames = 'domain'
+dover_1.ICPressure.Type = "HydroStaticPatch"
+dover_1.ICPressure.GeomNames = "domain"
 dover_1.Geom.domain.ICPressure.Value = -3.0
 
-dover_1.Geom.domain.ICPressure.RefGeom = 'domain'
-dover_1.Geom.domain.ICPressure.RefPatch = 'z_upper'
+dover_1.Geom.domain.ICPressure.RefGeom = "domain"
+dover_1.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dover_1.run()
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Testing serial run
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 
-dover_2 = dover_1.clone('dover_2')
+dover_2 = dover_1.clone("dover_2")
 
-dover_2.TopoSlopesX.Type = 'Constant'
-dover_2.TopoSlopesX.GeomNames = 'left right channel'
+dover_2.TopoSlopesX.Type = "Constant"
+dover_2.TopoSlopesX.GeomNames = "left right channel"
 dover_2.TopoSlopesX.Geom.left.Value = -0.002
 dover_2.TopoSlopesX.Geom.right.Value = 0.002
 dover_2.TopoSlopesX.Geom.channel.Value = 0.00
 
 dover_2.run()
-
diff --git a/test/python/new_features/simple-mask.py b/test/python/new_features/simple-mask.py
index b9c947ba1..ec084ab99 100644
--- a/test/python/new_features/simple-mask.py
+++ b/test/python/new_features/simple-mask.py
@@ -1,28 +1,32 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # example for pfsol generation
 #
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 from parflow import Run
 from parflow.tools.fs import get_absolute_path
-from parflow.tools.io import load_patch_matrix_from_sa_file, load_patch_matrix_from_asc_file, load_patch_matrix_from_image_file
+from parflow.tools.io import (
+    load_patch_matrix_from_sa_file,
+    load_patch_matrix_from_asc_file,
+    load_patch_matrix_from_image_file,
+)
 from parflow.tools.builders import SolidFileBuilder
 
 sabino = Run("sabino", __file__)
 
 sabino.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set Processor topology
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 sabino.Process.Topology.P = 1
 sabino.Process.Topology.Q = 1
 sabino.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 sabino.ComputationalGrid.Lower.X = 0.0
 sabino.ComputationalGrid.Lower.Y = 0.0
@@ -36,36 +40,37 @@
 sabino.ComputationalGrid.NY = 70
 sabino.ComputationalGrid.NZ = 20
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.GeomInput.Names = 'domaininput'
+sabino.GeomInput.Names = "domaininput"
 
-sabino.GeomInput.domaininput.GeomName = 'domain'
-sabino.GeomInput.domaininput.GeomNames = 'domain'
+sabino.GeomInput.domaininput.GeomName = "domain"
+sabino.GeomInput.domaininput.GeomNames = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # PFSOL generation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino_mask = load_patch_matrix_from_sa_file(get_absolute_path('$PF_SRC/test/input/Sabino_Mask.sa'))
+sabino_mask = load_patch_matrix_from_sa_file(
+    get_absolute_path("$PF_SRC/test/input/Sabino_Mask.sa")
+)
 # sabino_mask = load_patch_matrix_from_asc_file(get_absolute_path('$PF_SRC/test/input/Sabino_Mask.asc'))
 # sabino_mask = load_patch_matrix_from_image_file(get_absolute_path('$PF_SRC/test/input/Sabino_Mask.png'))
 # sabino_mask = load_patch_matrix_from_image_file(get_absolute_path('$PF_SRC/test/input/Sabino_Mask.tiff'))
 
-SolidFileBuilder(top=1, bottom=2, side=3) \
-  .mask(sabino_mask) \
-  .write('sabino_domain.pfsol', xllcorner=0, yllcorner=0, cellsize=90, vtk=True) \
-  .for_key(sabino.GeomInput.domaininput)
+SolidFileBuilder(top=1, bottom=2, side=3).mask(sabino_mask).write(
+    "sabino_domain.pfsol", xllcorner=0, yllcorner=0, cellsize=90, vtk=True
+).for_key(sabino.GeomInput.domaininput)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Geom.domain.Patches = 'top bottom edge'
+sabino.Geom.domain.Patches = "top bottom edge"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 sabino.Geom.domain.Lower.X = 0.0
 sabino.Geom.domain.Lower.Y = 0.0
@@ -75,180 +80,180 @@
 sabino.Geom.domain.Upper.Y = 6300.0
 sabino.Geom.domain.Upper.Z = 2000.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Permeability (values in m/hr)
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Geom.Perm.Names = 'domain'
+sabino.Geom.Perm.Names = "domain"
 
-sabino.Geom.domain.Perm.Type = 'Constant'
+sabino.Geom.domain.Perm.Type = "Constant"
 sabino.Geom.domain.Perm.Value = 0.0018
 
-sabino.Perm.TensorType = 'TensorByGeom'
-sabino.Geom.Perm.TensorByGeom.Names = 'domain'
+sabino.Perm.TensorType = "TensorByGeom"
+sabino.Geom.Perm.TensorByGeom.Names = "domain"
 sabino.Geom.domain.Perm.TensorValX = 1.0
 sabino.Geom.domain.Perm.TensorValY = 1.0
 sabino.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.SpecificStorage.Type = 'Constant'
-sabino.SpecificStorage.GeomNames = 'domain'
+sabino.SpecificStorage.Type = "Constant"
+sabino.SpecificStorage.GeomNames = "domain"
 sabino.Geom.domain.SpecificStorage.Value = 1.0e-5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Phase.Names = 'water'
-sabino.Phase.water.Density.Type = 'Constant'
+sabino.Phase.Names = "water"
+sabino.Phase.water.Density.Type = "Constant"
 sabino.Phase.water.Density.Value = 1.0
-sabino.Phase.water.Viscosity.Type = 'Constant'
+sabino.Phase.water.Viscosity.Type = "Constant"
 sabino.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Contaminants.Names = ''
+sabino.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 sabino.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Timing (time units is set by units of permeability)
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 sabino.TimingInfo.BaseUnit = 1.0
 sabino.TimingInfo.StartCount = 0
 sabino.TimingInfo.StartTime = 0.0
 sabino.TimingInfo.StopTime = 6.0
 sabino.TimingInfo.DumpInterval = 1.0
-sabino.TimeStep.Type = 'Constant'
+sabino.TimeStep.Type = "Constant"
 sabino.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Geom.Porosity.GeomNames = 'domain'
-sabino.Geom.domain.Porosity.Type = 'Constant'
+sabino.Geom.Porosity.GeomNames = "domain"
+sabino.Geom.domain.Porosity.Type = "Constant"
 sabino.Geom.domain.Porosity.Value = 0.1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Domain.GeomName = 'domain'
+sabino.Domain.GeomName = "domain"
 
-#----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 # Mobility
-#----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 
-sabino.Phase.water.Mobility.Type = 'Constant'
+sabino.Phase.water.Mobility.Type = "Constant"
 sabino.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Wells.Names = ''
+sabino.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Cycle.Names = 'constant'
-sabino.Cycle.constant.Names = 'alltime'
+sabino.Cycle.Names = "constant"
+sabino.Cycle.constant.Names = "alltime"
 sabino.Cycle.constant.alltime.Length = 1
 sabino.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.BCPressure.PatchNames = 'edge top bottom'
+sabino.BCPressure.PatchNames = "edge top bottom"
 
-sabino.Patch.edge.BCPressure.Type = 'FluxConst'
-sabino.Patch.edge.BCPressure.Cycle = 'constant'
+sabino.Patch.edge.BCPressure.Type = "FluxConst"
+sabino.Patch.edge.BCPressure.Cycle = "constant"
 sabino.Patch.edge.BCPressure.alltime.Value = 0.0
 
-sabino.Patch.bottom.BCPressure.Type = 'FluxConst'
-sabino.Patch.bottom.BCPressure.Cycle = 'constant'
+sabino.Patch.bottom.BCPressure.Type = "FluxConst"
+sabino.Patch.bottom.BCPressure.Cycle = "constant"
 sabino.Patch.bottom.BCPressure.alltime.Value = 0.0
 
 sabino.Patch.top.BCPressure.alltime.Value = -0.0035
-sabino.Patch.top.BCPressure.Type = 'OverlandFlow'
-sabino.Patch.top.BCPressure.Cycle = 'constant'
+sabino.Patch.top.BCPressure.Type = "OverlandFlow"
+sabino.Patch.top.BCPressure.Cycle = "constant"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Topo slopes in x-direction
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.TopoSlopesX.Type = 'Constant'
-sabino.TopoSlopesX.GeomNames = 'domain'
+sabino.TopoSlopesX.Type = "Constant"
+sabino.TopoSlopesX.GeomNames = "domain"
 sabino.TopoSlopesX.Geom.domain.Value = -0.005
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Topo slopes in y-direction
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.TopoSlopesY.Type = 'Constant'
-sabino.TopoSlopesY.GeomNames = 'domain'
+sabino.TopoSlopesY.Type = "Constant"
+sabino.TopoSlopesY.GeomNames = "domain"
 sabino.TopoSlopesY.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mannings coefficient
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Mannings.Type = 'Constant'
-sabino.Mannings.GeomNames = 'domain'
+sabino.Mannings.Type = "Constant"
+sabino.Mannings.GeomNames = "domain"
 sabino.Mannings.Geom.domain.Value = 5.52e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Phase.RelPerm.Type = 'VanGenuchten'
-sabino.Phase.RelPerm.GeomNames = 'domain'
+sabino.Phase.RelPerm.Type = "VanGenuchten"
+sabino.Phase.RelPerm.GeomNames = "domain"
 sabino.Geom.domain.RelPerm.Alpha = 3.5
 sabino.Geom.domain.RelPerm.N = 2.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Saturation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Phase.Saturation.Type = 'VanGenuchten'
-sabino.Phase.Saturation.GeomNames = 'domain'
+sabino.Phase.Saturation.Type = "VanGenuchten"
+sabino.Phase.Saturation.GeomNames = "domain"
 
 sabino.Geom.domain.Saturation.Alpha = 3.5
-sabino.Geom.domain.Saturation.N = 2.
+sabino.Geom.domain.Saturation.N = 2.0
 sabino.Geom.domain.Saturation.SRes = 0.1
 sabino.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.PhaseSources.water.Type = 'Constant'
-sabino.PhaseSources.water.GeomNames = 'domain'
+sabino.PhaseSources.water.Type = "Constant"
+sabino.PhaseSources.water.GeomNames = "domain"
 sabino.PhaseSources.water.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-sabino.ICPressure.Type = 'Constant'
-sabino.ICPressure.GeomNames = 'domain'
-sabino.Geom.domain.ICPressure.RefPatch = 'top'
+sabino.ICPressure.Type = "Constant"
+sabino.ICPressure.GeomNames = "domain"
+sabino.Geom.domain.ICPressure.RefPatch = "top"
 sabino.Geom.domain.ICPressure.Value = -10
 
-#----------------------------------------------------------------
+# ----------------------------------------------------------------
 # Outputs
 # ------------------------------------------------------------
 
@@ -257,39 +262,39 @@
 sabino.Solver.PrintSaturation = True
 sabino.Solver.PrintMask = True
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.KnownSolution = 'NoKnownSolution'
+sabino.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-sabino.Solver = 'Richards'
+sabino.Solver = "Richards"
 sabino.Solver.TerrainFollowingGrid = True
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setting up vertical layering
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 sabino.Solver.Nonlinear.VariableDz = True
-sabino.dzScale.GeomNames = 'domain'
-sabino.dzScale.Type = 'nzList'
+sabino.dzScale.GeomNames = "domain"
+sabino.dzScale.Type = "nzList"
 sabino.dzScale.nzListNumber = 20
 sabino.Cell._0.dzScale.Value = 2.0
 sabino.Cell._1.dzScale.Value = 2.0
 sabino.Cell._2.dzScale.Value = 2.0
 sabino.Cell._3.dzScale.Value = 1.0
 sabino.Cell._4.dzScale.Value = 1.0
-sabino.Cell._5.dzScale.Value = .50
-sabino.Cell._6.dzScale.Value = .50
-sabino.Cell._7.dzScale.Value = .20
-sabino.Cell._8.dzScale.Value = .20
-sabino.Cell._9.dzScale.Value = .20
-sabino.Cell._10.dzScale.Value = .20
-sabino.Cell._11.dzScale.Value = .10
+sabino.Cell._5.dzScale.Value = 0.50
+sabino.Cell._6.dzScale.Value = 0.50
+sabino.Cell._7.dzScale.Value = 0.20
+sabino.Cell._8.dzScale.Value = 0.20
+sabino.Cell._9.dzScale.Value = 0.20
+sabino.Cell._10.dzScale.Value = 0.20
+sabino.Cell._11.dzScale.Value = 0.10
 sabino.Cell._12.dzScale.Value = 0.02
 sabino.Cell._13.dzScale.Value = 0.02
 sabino.Cell._14.dzScale.Value = 0.02
@@ -300,26 +305,26 @@
 sabino.Cell._19.dzScale.Value = 0.001
 
 sabino.Solver.MaxIter = 25000000
-sabino.Solver.Drop = 1E-20
-sabino.Solver.AbsTol = 1E-8
+sabino.Solver.Drop = 1e-20
+sabino.Solver.AbsTol = 1e-8
 sabino.Solver.MaxConvergenceFailures = 8
 sabino.Solver.Nonlinear.MaxIter = 80
 sabino.Solver.Nonlinear.ResidualTol = 1e-6
 
-sabino.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+sabino.Solver.Nonlinear.EtaChoice = "EtaConstant"
 sabino.Solver.Nonlinear.EtaValue = 0.001
 sabino.Solver.Nonlinear.UseJacobian = True
 sabino.Solver.Nonlinear.DerivativeEpsilon = 1e-16
 sabino.Solver.Nonlinear.StepTol = 1e-30
-sabino.Solver.Nonlinear.Globalization = 'LineSearch'
+sabino.Solver.Nonlinear.Globalization = "LineSearch"
 sabino.Solver.Linear.KrylovDimension = 70
 sabino.Solver.Linear.MaxRestarts = 2
 
-sabino.Solver.Linear.Preconditioner = 'PFMG'
-sabino.Solver.Linear.Preconditioner.PCMatrixType = 'FullJacobian'
+sabino.Solver.Linear.Preconditioner = "PFMG"
+sabino.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Distribute inputs and run simulation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 sabino.run()
diff --git a/test/python/new_features/table_loading.py b/test/python/new_features/table_loading.py
index 417dc9a17..24ca268e8 100644
--- a/test/python/new_features/table_loading.py
+++ b/test/python/new_features/table_loading.py
@@ -1,12 +1,11 @@
-
 from parflow import Run
 from parflow.tools.builders import SubsurfacePropertiesBuilder
 
 run = Run("table_test", __file__)
 
-run.GeomInput.Names = 's1 s2 s3 s4'
+run.GeomInput.Names = "s1 s2 s3 s4"
 
-soil_properties = '''
+soil_properties = """
 # -----------------------------------------------------------------------------
 # Sample header
 # -----------------------------------------------------------------------------
@@ -24,26 +23,26 @@
 s5   0.65  0.675     0.455            5.5      4.5       5.6      9
 
 s6   -     0.7       -                6.6      -         -        -
-'''
+"""
 
 soil_builder = SubsurfacePropertiesBuilder(run).load_txt_content(soil_properties)
 
-print('-'*80)
-print('basic mapping')
-print('-'*80)
+print("-" * 80)
+print("basic mapping")
+print("-" * 80)
 soil_builder.print()
-print('-'*80)
-print('remap s5 to s4')
-print('-'*80)
-soil_builder.assign('s5', 's4').print()
+print("-" * 80)
+print("remap s5 to s4")
+print("-" * 80)
+soil_builder.assign("s5", "s4").print()
 
-print('-'*80)
-print('remap s6 to s4 (with skip)')
-print('-'*80)
-soil_builder.assign('s6', 's4').print()
+print("-" * 80)
+print("remap s6 to s4 (with skip)")
+print("-" * 80)
+soil_builder.assign("s6", "s4").print()
 
-print('+'*80)
-soil_properties_transpose = '''
+print("+" * 80)
+soil_properties_transpose = """
 # -----------------------------------------------------------------------------
 # Sample header transposed
 # -----------------------------------------------------------------------------
@@ -56,21 +55,23 @@
 RelPermN      1.4     1.7    2.3      3.4     5.6     -
 A             5       6      7        8       9       -
 # -----------------------------------------------------------------------------
-'''
-soil_builder = SubsurfacePropertiesBuilder(run).load_txt_content(soil_properties_transpose)
+"""
+soil_builder = SubsurfacePropertiesBuilder(run).load_txt_content(
+    soil_properties_transpose
+)
 
-print('-'*80)
-print('basic mapping')
-print('-'*80)
+print("-" * 80)
+print("basic mapping")
+print("-" * 80)
 soil_builder.print()
-print('-'*80)
-print('remap s5 to s4')
-print('-'*80)
-soil_builder.assign('s5', 's4').print()
-print('-'*80)
-print('remap s6 to s4 (with skip)')
-print('-'*80)
-soil_builder.assign('s6', 's4').print()
+print("-" * 80)
+print("remap s5 to s4")
+print("-" * 80)
+soil_builder.assign("s5", "s4").print()
+print("-" * 80)
+print("remap s6 to s4 (with skip)")
+print("-" * 80)
+soil_builder.assign("s6", "s4").print()
 
 soil_builder.apply()
 
diff --git a/test/python/new_features/tables_LW.py b/test/python/new_features/tables_LW.py
index d5d3e8b6f..f0168a19f 100644
--- a/test/python/new_features/tables_LW.py
+++ b/test/python/new_features/tables_LW.py
@@ -11,17 +11,17 @@
 
 LW_Test.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set Processor topology
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_Test.Process.Topology.P = 1
 LW_Test.Process.Topology.Q = 1
 LW_Test.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_Test.ComputationalGrid.Lower.X = 0.0
 LW_Test.ComputationalGrid.Lower.Y = 0.0
@@ -35,22 +35,22 @@
 LW_Test.ComputationalGrid.NY = 41
 LW_Test.ComputationalGrid.NZ = 50
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.GeomInput.Names = 'box_input indi_input'
+LW_Test.GeomInput.Names = "box_input indi_input"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.GeomInput.box_input.InputType = 'Box'
-LW_Test.GeomInput.box_input.GeomName = 'domain'
+LW_Test.GeomInput.box_input.InputType = "Box"
+LW_Test.GeomInput.box_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_Test.Geom.domain.Lower.X = 0.0
 LW_Test.Geom.domain.Lower.Y = 0.0
@@ -59,15 +59,17 @@
 LW_Test.Geom.domain.Upper.X = 41000.0
 LW_Test.Geom.domain.Upper.Y = 41000.0
 LW_Test.Geom.domain.Upper.Z = 100.0
-LW_Test.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+LW_Test.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Indicator Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.GeomInput.indi_input.InputType = 'IndicatorField'
-LW_Test.GeomInput.indi_input.GeomNames = 's1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 g1 g2 g3 g4 g5 g6 g7 g8'
-LW_Test.Geom.indi_input.FileName = 'IndicatorFile_Gleeson.50z.pfb'
+LW_Test.GeomInput.indi_input.InputType = "IndicatorField"
+LW_Test.GeomInput.indi_input.GeomNames = (
+    "s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 g1 g2 g3 g4 g5 g6 g7 g8"
+)
+LW_Test.Geom.indi_input.FileName = "IndicatorFile_Gleeson.50z.pfb"
 
 LW_Test.GeomInput.s1.Value = 1
 LW_Test.GeomInput.s2.Value = 2
@@ -91,11 +93,11 @@
 LW_Test.GeomInput.g7.Value = 27
 LW_Test.GeomInput.g8.Value = 28
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Permeability (values in m/hr)
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_subsurface_properties = '''
+LW_subsurface_properties = """
 key     Perm          Porosity   RelPermAlpha  RelPermN  SatAlpha  SatN    SatSRes    SatSSat
 domain  0.2           0.4        3.5           2         3.5       2       0.2        1
 s1      0.269022595   0.375      3.548         4.162     3.548     4.162   0.000001   1
@@ -111,172 +113,170 @@
 g3      0.059         -          -             -         -         -       -          -
 g6      0.2           -          -             -         -         -       -          -
 g8      0.68          -          -             -         -         -       -          -
-'''
+"""
 
 # standard inline procedure calls in Python
-SubsurfacePropertiesBuilder(LW_Test)\
-  .load_txt_content(LW_subsurface_properties) \
-  .apply()
+SubsurfacePropertiesBuilder(LW_Test).load_txt_content(LW_subsurface_properties).apply()
 
 # alternative option separating method calls
 # spb = SubsurfacePropertiesBuilder(LW_Test)
 # spb.load_txt_content(LW_subsurface_properties)
 # spb.apply(name_registration=True)
 
-LW_Test.Perm.TensorType = 'TensorByGeom'
-LW_Test.Geom.Perm.TensorByGeom.Names = 'domain'
+LW_Test.Perm.TensorType = "TensorByGeom"
+LW_Test.Geom.Perm.TensorByGeom.Names = "domain"
 LW_Test.Geom.domain.Perm.TensorValX = 1.0
 LW_Test.Geom.domain.Perm.TensorValY = 1.0
 LW_Test.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.SpecificStorage.Type = 'Constant'
-LW_Test.SpecificStorage.GeomNames = 'domain'
+LW_Test.SpecificStorage.Type = "Constant"
+LW_Test.SpecificStorage.GeomNames = "domain"
 LW_Test.Geom.domain.SpecificStorage.Value = 1.0e-5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.Phase.Names = 'water'
-LW_Test.Phase.water.Density.Type = 'Constant'
+LW_Test.Phase.Names = "water"
+LW_Test.Phase.water.Density.Type = "Constant"
 LW_Test.Phase.water.Density.Value = 1.0
-LW_Test.Phase.water.Viscosity.Type = 'Constant'
+LW_Test.Phase.water.Viscosity.Type = "Constant"
 LW_Test.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.Contaminants.Names = ''
+LW_Test.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_Test.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Timing (time units is set by units of permeability)
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_Test.TimingInfo.BaseUnit = 1.0
 LW_Test.TimingInfo.StartCount = 0
 LW_Test.TimingInfo.StartTime = 0.0
 LW_Test.TimingInfo.StopTime = 1000.0
 LW_Test.TimingInfo.DumpInterval = 1.0
-LW_Test.TimeStep.Type = 'Constant'
+LW_Test.TimeStep.Type = "Constant"
 LW_Test.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.Domain.GeomName = 'domain'
+LW_Test.Domain.GeomName = "domain"
 
-#----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 # Mobility
-#----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 
-LW_Test.Phase.water.Mobility.Type = 'Constant'
+LW_Test.Phase.water.Mobility.Type = "Constant"
 LW_Test.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.Wells.Names = ''
+LW_Test.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.Cycle.Names = 'constant rainrec'
-LW_Test.Cycle.constant.Names = 'alltime'
+LW_Test.Cycle.Names = "constant rainrec"
+LW_Test.Cycle.constant.Names = "alltime"
 LW_Test.Cycle.constant.alltime.Length = 1
 LW_Test.Cycle.constant.Repeat = -1
 
-LW_Test.Cycle.rainrec.Names = 'rain rec'
+LW_Test.Cycle.rainrec.Names = "rain rec"
 LW_Test.Cycle.rainrec.rain.Length = 10
 LW_Test.Cycle.rainrec.rec.Length = 150
 LW_Test.Cycle.rainrec.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_Test.BCPressure.PatchNames = LW_Test.Geom.domain.Patches
 
-LW_Test.Patch.x_lower.BCPressure.Type = 'FluxConst'
-LW_Test.Patch.x_lower.BCPressure.Cycle = 'constant'
+LW_Test.Patch.x_lower.BCPressure.Type = "FluxConst"
+LW_Test.Patch.x_lower.BCPressure.Cycle = "constant"
 LW_Test.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-LW_Test.Patch.y_lower.BCPressure.Type = 'FluxConst'
-LW_Test.Patch.y_lower.BCPressure.Cycle = 'constant'
+LW_Test.Patch.y_lower.BCPressure.Type = "FluxConst"
+LW_Test.Patch.y_lower.BCPressure.Cycle = "constant"
 LW_Test.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-LW_Test.Patch.z_lower.BCPressure.Type = 'FluxConst'
-LW_Test.Patch.z_lower.BCPressure.Cycle = 'constant'
+LW_Test.Patch.z_lower.BCPressure.Type = "FluxConst"
+LW_Test.Patch.z_lower.BCPressure.Cycle = "constant"
 LW_Test.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-LW_Test.Patch.x_upper.BCPressure.Type = 'FluxConst'
-LW_Test.Patch.x_upper.BCPressure.Cycle = 'constant'
+LW_Test.Patch.x_upper.BCPressure.Type = "FluxConst"
+LW_Test.Patch.x_upper.BCPressure.Cycle = "constant"
 LW_Test.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-LW_Test.Patch.y_upper.BCPressure.Type = 'FluxConst'
-LW_Test.Patch.y_upper.BCPressure.Cycle = 'constant'
+LW_Test.Patch.y_upper.BCPressure.Type = "FluxConst"
+LW_Test.Patch.y_upper.BCPressure.Cycle = "constant"
 LW_Test.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-LW_Test.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-LW_Test.Patch.z_upper.BCPressure.Cycle = 'rainrec'
+LW_Test.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+LW_Test.Patch.z_upper.BCPressure.Cycle = "rainrec"
 LW_Test.Patch.z_upper.BCPressure.rain.Value = -0.1
 LW_Test.Patch.z_upper.BCPressure.rec.Value = 0.0000
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Topo slopes in x-direction
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.TopoSlopesX.Type = 'PFBFile'
-LW_Test.TopoSlopesX.GeomNames = 'domain'
-LW_Test.TopoSlopesX.FileName = 'LW.slopex.pfb'
+LW_Test.TopoSlopesX.Type = "PFBFile"
+LW_Test.TopoSlopesX.GeomNames = "domain"
+LW_Test.TopoSlopesX.FileName = "LW.slopex.pfb"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Topo slopes in y-direction
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.TopoSlopesY.Type = 'PFBFile'
-LW_Test.TopoSlopesY.GeomNames = 'domain'
-LW_Test.TopoSlopesY.FileName = 'LW.slopey.pfb'
+LW_Test.TopoSlopesY.Type = "PFBFile"
+LW_Test.TopoSlopesY.GeomNames = "domain"
+LW_Test.TopoSlopesY.FileName = "LW.slopey.pfb"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mannings coefficient
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.Mannings.Type = 'Constant'
-LW_Test.Mannings.GeomNames = 'domain'
+LW_Test.Mannings.Type = "Constant"
+LW_Test.Mannings.GeomNames = "domain"
 LW_Test.Mannings.Geom.domain.Value = 5.52e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.PhaseSources.water.Type = 'Constant'
-LW_Test.PhaseSources.water.GeomNames = 'domain'
+LW_Test.PhaseSources.water.Type = "Constant"
+LW_Test.PhaseSources.water.GeomNames = "domain"
 LW_Test.PhaseSources.water.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LW_Test.ICPressure.Type = 'PFBFile'
-LW_Test.ICPressure.GeomNames = 'domain'
-LW_Test.Geom.domain.ICPressure.RefPatch = 'z_upper'
-LW_Test.Geom.domain.ICPressure.FileName = 'press.init.pfb'
+LW_Test.ICPressure.Type = "PFBFile"
+LW_Test.ICPressure.GeomNames = "domain"
+LW_Test.Geom.domain.ICPressure.RefPatch = "z_upper"
+LW_Test.Geom.domain.ICPressure.FileName = "press.init.pfb"
 
-#----------------------------------------------------------------
+# ----------------------------------------------------------------
 # Outputs
 # ---------------------------------------------------------------
 
@@ -285,135 +285,137 @@
 LW_Test.Solver.PrintSaturation = True
 LW_Test.Solver.PrintMask = True
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.KnownSolution = 'NoKnownSolution'
+LW_Test.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 # ParFlow Solution
-LW_Test.Solver = 'Richards'
+LW_Test.Solver = "Richards"
 LW_Test.Solver.TerrainFollowingGrid = True
 LW_Test.Solver.Nonlinear.VariableDz = False
 
 LW_Test.Solver.MaxIter = 25000
-LW_Test.Solver.Drop = 1E-20
-LW_Test.Solver.AbsTol = 1E-8
+LW_Test.Solver.Drop = 1e-20
+LW_Test.Solver.AbsTol = 1e-8
 LW_Test.Solver.MaxConvergenceFailures = 8
 LW_Test.Solver.Nonlinear.MaxIter = 80
 LW_Test.Solver.Nonlinear.ResidualTol = 1e-6
 
-LW_Test.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+LW_Test.Solver.Nonlinear.EtaChoice = "EtaConstant"
 LW_Test.Solver.Nonlinear.EtaValue = 0.001
 LW_Test.Solver.Nonlinear.UseJacobian = True
 LW_Test.Solver.Nonlinear.DerivativeEpsilon = 1e-16
 LW_Test.Solver.Nonlinear.StepTol = 1e-30
-LW_Test.Solver.Nonlinear.Globalization = 'LineSearch'
+LW_Test.Solver.Nonlinear.Globalization = "LineSearch"
 LW_Test.Solver.Linear.KrylovDimension = 70
 LW_Test.Solver.Linear.MaxRestarts = 2
 
-LW_Test.Solver.Linear.Preconditioner = 'PFMG'
+LW_Test.Solver.Linear.Preconditioner = "PFMG"
+
 
 def test_subsurface_table(file_name):
-    path = '$PF_SRC/test/correct_output/tables_LW_subsurface.txt.ref'
+    path = "$PF_SRC/test/correct_output/tables_LW_subsurface.txt.ref"
     ref = Path(get_absolute_path(path)).read_text()
     new = Path(get_absolute_path(file_name)).read_text()
     assert ref == new
 
-#-----------------------------------------------------------------------------
+
+# -----------------------------------------------------------------------------
 # Comparing written key/value pairs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+
 
 def test_output(file_name):
-  LW_Test.write(file_name, file_format='yaml')
-  with open(get_absolute_path(f'{file_name}.yaml')) as new, \
-      open(get_absolute_path('$PF_SRC/test/correct_output/LW_test_ref.yaml.ref')) as ref:
-    if new.read() == ref.read():
-      print('Success we have the same file')
-      return True
-    else:
-      print('Files are different')
-      return False
+    LW_Test.write(file_name, file_format="yaml")
+    with open(get_absolute_path(f"{file_name}.yaml")) as new, open(
+        get_absolute_path("$PF_SRC/test/correct_output/LW_test_ref.yaml.ref")
+    ) as ref:
+        if new.read() == ref.read():
+            print("Success we have the same file")
+            return True
+        else:
+            print("Files are different")
+            return False
+
 
 # testing inline text
-print('+'*40)
-print('Comparing inline text to ref:')
+print("+" * 40)
+print("Comparing inline text to ref:")
 
-LW_Test.write_subsurface_table('inline_input.txt')
-test_subsurface_table('inline_input.txt')
+LW_Test.write_subsurface_table("inline_input.txt")
+test_subsurface_table("inline_input.txt")
 
-if not test_output('inline_input_yaml'):
-  sys.exit(1)
-print('')
+if not test_output("inline_input_yaml"):
+    sys.exit(1)
+print("")
 
 # resetting properties to zero
-print('+'*40)
-print('Clearing:')
-SubsurfacePropertiesBuilder(LW_Test) \
-  .load_csv_file('$PF_SRC/test/input/LW_test_data_clear.csv') \
-  .apply()
-if test_output('clear_yaml'): # should fail
-  sys.exit(1)
-print('+'*40)
-print('')
+print("+" * 40)
+print("Clearing:")
+SubsurfacePropertiesBuilder(LW_Test).load_csv_file(
+    "$PF_SRC/test/input/LW_test_data_clear.csv"
+).apply()
+if test_output("clear_yaml"):  # should fail
+    sys.exit(1)
+print("+" * 40)
+print("")
 
 # testing csv file
-print('Loading csv file:')
-SubsurfacePropertiesBuilder(LW_Test) \
-  .load_csv_file('$PF_SRC/test/input/LW_test_data.csv') \
-  .apply() \
-  .print_as_table(props_in_header=False)
-print('+'*40)
-print('Comparing csv file to ref:')
-if not test_output('csv_input_yaml'):
-  sys.exit(1)
-print('+' * 40)
-print('')
+print("Loading csv file:")
+SubsurfacePropertiesBuilder(LW_Test).load_csv_file(
+    "$PF_SRC/test/input/LW_test_data.csv"
+).apply().print_as_table(props_in_header=False)
+print("+" * 40)
+print("Comparing csv file to ref:")
+if not test_output("csv_input_yaml"):
+    sys.exit(1)
+print("+" * 40)
+print("")
 
 # resetting properties to zero
-print('Clearing:')
-SubsurfacePropertiesBuilder(LW_Test) \
-  .load_csv_file('$PF_SRC/test/input/LW_test_data_clear.csv') \
-  .apply()
-if test_output('clear_yaml'): # should fail
-  sys.exit(1)
-print('+'*40)
-print('')
+print("Clearing:")
+SubsurfacePropertiesBuilder(LW_Test).load_csv_file(
+    "$PF_SRC/test/input/LW_test_data_clear.csv"
+).apply()
+if test_output("clear_yaml"):  # should fail
+    sys.exit(1)
+print("+" * 40)
+print("")
 
 # testing txt file
-print('Loading txt file:')
-SubsurfacePropertiesBuilder(LW_Test) \
-    .load_txt_file('$PF_SRC/test/input/LW_test_data.txt') \
-    .apply() \
-    .print_as_table()
-print('+'*40)
-print('Comparing txt file to ref:')
-if not test_output('txt_input_yaml'):
-  sys.exit(1)
-print('+' * 40)
+print("Loading txt file:")
+SubsurfacePropertiesBuilder(LW_Test).load_txt_file(
+    "$PF_SRC/test/input/LW_test_data.txt"
+).apply().print_as_table()
+print("+" * 40)
+print("Comparing txt file to ref:")
+if not test_output("txt_input_yaml"):
+    sys.exit(1)
+print("+" * 40)
 
 # resetting properties to zero
-print('Clearing:')
-SubsurfacePropertiesBuilder(LW_Test) \
-  .load_csv_file('$PF_SRC/test/input/LW_test_data_clear.csv') \
-  .apply()
-if test_output('clear_yaml'): # should fail
-  sys.exit(1)
-print('+'*40)
-print('')
+print("Clearing:")
+SubsurfacePropertiesBuilder(LW_Test).load_csv_file(
+    "$PF_SRC/test/input/LW_test_data_clear.csv"
+).apply()
+if test_output("clear_yaml"):  # should fail
+    sys.exit(1)
+print("+" * 40)
+print("")
 
 # testing txt file
-print('Loading transposed txt file:')
-SubsurfacePropertiesBuilder(LW_Test) \
-    .load_txt_file('$PF_SRC/test/input/LW_test_data_transposed.txt') \
-    .apply() \
-    .print()
-print('+'*40)
-print('Comparing transposed txt file to ref:')
-if not test_output('trasnposed_txt_input_yaml'):
-  sys.exit(1)
-print('+' * 40)
+print("Loading transposed txt file:")
+SubsurfacePropertiesBuilder(LW_Test).load_txt_file(
+    "$PF_SRC/test/input/LW_test_data_transposed.txt"
+).apply().print()
+print("+" * 40)
+print("Comparing transposed txt file to ref:")
+if not test_output("trasnposed_txt_input_yaml"):
+    sys.exit(1)
+print("+" * 40)
diff --git a/test/python/new_features/validation_print.py b/test/python/new_features/validation_print.py
index fc4063e8e..a62932a9c 100644
--- a/test/python/new_features/validation_print.py
+++ b/test/python/new_features/validation_print.py
@@ -1,21 +1,21 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  Same as default_richards.py run, with several errors to
 #  demonstrate the skip_valid argument in validate
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 from parflow import Run
 
-drich = Run('default_richards', __file__)
+drich = Run("default_richards", __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Process.Topology.P = 1.5
 drich.Process.Topology.Q = 1
 drich.Process.Topology.R = -1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.ComputationalGrid.Lower.X = -10.0
 drich.ComputationalGrid.Lower.Y = 10.0
@@ -29,22 +29,24 @@
 drich.ComputationalGrid.NY = 10
 drich.ComputationalGrid.NZ = 8
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.Names = "domain_input background_input source_region_input concen_region_input"
+drich.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.domain_input.InputType = 'Box'
-drich.GeomInput.domain_input.GeomName = 'domain'
+drich.GeomInput.domain_input.InputType = "Box"
+drich.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Geom.domain.Lower.X = -10.0
 drich.Geom.domain.Lower.Y = 10.0
@@ -56,16 +58,16 @@
 
 drich.Geom.domain.Patches = "left right front back bottom top"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.background_input.InputType = 'Box'
-drich.GeomInput.background_input.GeomName = 'background'
+drich.GeomInput.background_input.InputType = "Box"
+drich.GeomInput.background_input.GeomName = "background"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Geom.background.Lower.X = -99999999.0
 drich.Geom.background.Lower.Y = -99999999.0
@@ -75,16 +77,16 @@
 drich.Geom.background.Upper.Y = 99999999.0
 drich.Geom.background.Upper.Z = 99999999.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.source_region_input.InputType = 'Box'
-drich.GeomInput.source_region_input.GeomName = 'source_region'
+drich.GeomInput.source_region_input.InputType = "Box"
+drich.GeomInput.source_region_input.GeomName = "source_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Geom.source_region.Lower.X = 65.56
 drich.Geom.source_region.Lower.Y = 79.34
@@ -94,16 +96,16 @@
 drich.Geom.source_region.Upper.Y = 89.99
 drich.Geom.source_region.Upper.Z = 5.5
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.GeomInput.concen_region_input.InputType = 'Box'
-drich.GeomInput.concen_region_input.GeomName = 'concen_region'
+drich.GeomInput.concen_region_input.InputType = "Box"
+drich.GeomInput.concen_region_input.GeomName = "concen_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Geom.concen_region.Lower.X = 60.0
 drich.Geom.concen_region.Lower.Y = 80.0
@@ -113,16 +115,16 @@
 drich.Geom.concen_region.Upper.Y = 100.0
 drich.Geom.concen_region.Upper.Z = 6.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.Geom.Perm.Names = "background"
 
-drich.Geom.background.Perm.Type = 'Constant'
+drich.Geom.background.Perm.Type = "Constant"
 drich.Geom.background.Perm.Value = 4.0
 
-drich.Perm.TensorType = 'TensorByGeom'
+drich.Perm.TensorType = "TensorByGeom"
 
 drich.Geom.Perm.TensorByGeom.Names = "background"
 
@@ -130,207 +132,207 @@
 drich.Geom.background.Perm.TensorValY = 1.0
 drich.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.SpecificStorage.Type = 'Constant'
+drich.SpecificStorage.Type = "Constant"
 drich.SpecificStorage.GeomNames = "domain"
 drich.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.Phase.Names = "water"
 
-drich.Phase.water.Density.Type = 'Constant'
+drich.Phase.water.Density.Type = "Constant"
 drich.Phase.water.Density.Value = 1.0
 
-drich.Phase.water.Viscosity.Type = 'Constant'
+drich.Phase.water.Viscosity.Type = "Constant"
 drich.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.TimingInfo.BaseUnit = 1.0
 drich.TimingInfo.StartCount = 0
 drich.TimingInfo.StartTime = 0.0
 drich.TimingInfo.StopTime = 0.010
 drich.TimingInfo.DumpInterval = -1
-drich.TimeStep.Type = 'Random'
+drich.TimeStep.Type = "Random"
 drich.TimeStep.Value = 0.001
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Geom.Porosity.GeomNames = 'background'
-drich.Geom.background.Porosity.Type = 'Constant'
+drich.Geom.Porosity.GeomNames = "background"
+drich.Geom.background.Porosity.Type = "Constant"
 drich.Geom.background.Porosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Domain.GeomName = 'domain'
+drich.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Phase.RelPerm.Type = 'VanGenuchten'
-drich.Phase.RelPerm.GeomNames = 'domain'
+drich.Phase.RelPerm.Type = "VanGenuchten"
+drich.Phase.RelPerm.GeomNames = "domain"
 drich.Geom.domain.RelPerm.Alpha = 0.005
 drich.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.Phase.Saturation.Type = 'VanGenuchten'
-drich.Phase.Saturation.GeomNames = 'domain'
+drich.Phase.Saturation.Type = "VanGenuchten"
+drich.Phase.Saturation.GeomNames = "domain"
 drich.Geom.domain.Saturation.Alpha = 0.005
 drich.Geom.domain.Saturation.N = 2.0
 drich.Geom.domain.Saturation.SRes = 0.2
 drich.Geom.domain.Saturation.SSat = 1.99
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Wells.Names = ''
+drich.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Cycle.Names = 'constant'
+drich.Cycle.Names = "constant"
 drich.Cycle.constant.Names = "alltime"
 drich.Cycle.constant.alltime.Length = 1
 drich.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.BCPressure.PatchNames = "left right front back bottom top"
 
-drich.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
+drich.Patch.left.BCPressure.Type = "DirEquilRefPatch"
 drich.Patch.left.BCPressure.Cycle = "constant"
-drich.Patch.left.BCPressure.RefGeom = 'domain'
-drich.Patch.left.BCPressure.RefPatch = 'bottom'
+drich.Patch.left.BCPressure.RefGeom = "domain"
+drich.Patch.left.BCPressure.RefPatch = "bottom"
 drich.Patch.left.BCPressure.alltime.Value = 5.0
 
-drich.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
+drich.Patch.right.BCPressure.Type = "DirEquilRefPatch"
 drich.Patch.right.BCPressure.Cycle = "constant"
-drich.Patch.right.BCPressure.RefGeom = 'domain'
-drich.Patch.right.BCPressure.RefPatch = 'bottom'
+drich.Patch.right.BCPressure.RefGeom = "domain"
+drich.Patch.right.BCPressure.RefPatch = "bottom"
 drich.Patch.right.BCPressure.alltime.Value = 3.0
 
-drich.Patch.front.BCPressure.Type = 'FluxConst'
+drich.Patch.front.BCPressure.Type = "FluxConst"
 drich.Patch.front.BCPressure.Cycle = "constant"
 drich.Patch.front.BCPressure.alltime.Value = 0.0
 
-drich.Patch.back.BCPressure.Type = 'FluxConst'
+drich.Patch.back.BCPressure.Type = "FluxConst"
 drich.Patch.back.BCPressure.Cycle = "constant"
 drich.Patch.back.BCPressure.alltime.Value = 0.0
 
-drich.Patch.bottom.BCPressure.Type = 'FluxConst'
+drich.Patch.bottom.BCPressure.Type = "FluxConst"
 drich.Patch.bottom.BCPressure.Cycle = "constant"
 drich.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-drich.Patch.top.BCPressure.Type = 'FluxConst'
+drich.Patch.top.BCPressure.Type = "FluxConst"
 drich.Patch.top.BCPressure.Cycle = "constant"
 drich.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.TopoSlopesX.Type = "Constant"
 drich.TopoSlopesX.GeomNames = "domain"
 drich.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.TopoSlopesY.Type = "Constant"
 drich.TopoSlopesY.GeomNames = "domain"
 drich.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 drich.Mannings.Type = "Constant"
 drich.Mannings.GeomNames = "domain"
-drich.Mannings.Geom.domain.Value = 0.
+drich.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-drich.ICPressure.Type = 'HydroStaticPatch'
-drich.ICPressure.GeomNames = 'domain'
+drich.ICPressure.Type = "HydroStaticPatch"
+drich.ICPressure.GeomNames = "domain"
 drich.Geom.domain.ICPressure.Value = 3.0
-drich.Geom.domain.ICPressure.RefGeom = 'domain'
-drich.Geom.domain.ICPressure.RefPatch = 'bottom'
+drich.Geom.domain.ICPressure.RefGeom = "domain"
+drich.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.PhaseSources.water.Type = 'Constant'
-drich.PhaseSources.water.GeomNames = 'background'
+drich.PhaseSources.water.Type = "Constant"
+drich.PhaseSources.water.GeomNames = "background"
 drich.PhaseSources.water.Geom.background.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.KnownSolution = 'NoKnownSolution'
+drich.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-drich.Solver = 'Richards'
+drich.Solver = "Richards"
 drich.Solver.MaxIter = 5
 
 drich.Solver.Nonlinear.MaxIter = -10
 drich.Solver.Nonlinear.ResidualTol = 1e-9
-drich.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+drich.Solver.Nonlinear.EtaChoice = "EtaConstant"
 drich.Solver.Nonlinear.EtaValue = 1e-5
 drich.Solver.Nonlinear.UseJacobian = True
 drich.Solver.Nonlinear.DerivativeEpsilon = 1e-2
 
 drich.Solver.Linear.KrylovDimension = 10
 
-drich.Solver.Linear.Preconditioner = 'PFMG'
+drich.Solver.Linear.Preconditioner = "PFMG"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run ParFlow
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 drich.validate(verbose=False)
diff --git a/test/python/new_features/write_check.py b/test/python/new_features/write_check.py
index 00fa2c061..e73f98e04 100644
--- a/test/python/new_features/write_check.py
+++ b/test/python/new_features/write_check.py
@@ -1,6 +1,6 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Testing file writing
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 import sys
 from parflow import Run
@@ -8,22 +8,22 @@
 
 dsingle = Run("dsingle", __file__)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # File input version number
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 dsingle.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Process Topology
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dsingle.Process.Topology.P = 1
 dsingle.Process.Topology.Q = 1
 dsingle.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 dsingle.ComputationalGrid.Lower.X = -10.0
 dsingle.ComputationalGrid.Lower.Y = 10.0
 dsingle.ComputationalGrid.Lower.Z = 1.0
@@ -36,21 +36,23 @@
 dsingle.ComputationalGrid.NY = 15
 dsingle.ComputationalGrid.NZ = 8
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # The Names of the GeomInputs
-#-----------------------------------------------------------------------------
-dsingle.GeomInput.Names = 'domain_input background_input source_region_input concen_region_input'
+# -----------------------------------------------------------------------------
+dsingle.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
-dsingle.GeomInput.domain_input.InputType = 'Box'
-dsingle.GeomInput.domain_input.GeomName = 'domain'
+# -----------------------------------------------------------------------------
+dsingle.GeomInput.domain_input.InputType = "Box"
+dsingle.GeomInput.domain_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 dsingle.Geom.domain.Lower.X = -10.0
 dsingle.Geom.domain.Lower.Y = 10.0
 dsingle.Geom.domain.Lower.Z = 1.0
@@ -59,17 +61,17 @@
 dsingle.Geom.domain.Upper.Y = 170.0
 dsingle.Geom.domain.Upper.Z = 9.0
 
-dsingle.Geom.domain.Patches = 'left right front back bottom top'
+dsingle.Geom.domain.Patches = "left right front back bottom top"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Background Geometry Input
-#-----------------------------------------------------------------------------
-dsingle.GeomInput.background_input.InputType = 'Box'
-dsingle.GeomInput.background_input.GeomName = 'background'
+# -----------------------------------------------------------------------------
+dsingle.GeomInput.background_input.InputType = "Box"
+dsingle.GeomInput.background_input.GeomName = "background"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Background Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 dsingle.Geom.background.Lower.X = -99999999.0
 dsingle.Geom.background.Lower.Y = -99999999.0
 dsingle.Geom.background.Lower.Z = -99999999.0
@@ -79,15 +81,15 @@
 dsingle.Geom.background.Upper.Z = 99999999.0
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Source_Region Geometry Input
-#-----------------------------------------------------------------------------
-dsingle.GeomInput.source_region_input.InputType = 'Box'
-dsingle.GeomInput.source_region_input.GeomName = 'source_region'
+# -----------------------------------------------------------------------------
+dsingle.GeomInput.source_region_input.InputType = "Box"
+dsingle.GeomInput.source_region_input.GeomName = "source_region"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Source_Region Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 dsingle.Geom.source_region.Lower.X = 65.56
 dsingle.Geom.source_region.Lower.Y = 79.34
 dsingle.Geom.source_region.Lower.Z = 4.5
@@ -97,15 +99,15 @@
 dsingle.Geom.source_region.Upper.Z = 5.5
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Concen_Region Geometry Input
-#-----------------------------------------------------------------------------
-dsingle.GeomInput.concen_region_input.InputType = 'Box'
-dsingle.GeomInput.concen_region_input.GeomName = 'concen_region'
+# -----------------------------------------------------------------------------
+dsingle.GeomInput.concen_region_input.InputType = "Box"
+dsingle.GeomInput.concen_region_input.GeomName = "concen_region"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Concen_Region Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 dsingle.Geom.concen_region.Lower.X = 60.0
 dsingle.Geom.concen_region.Lower.Y = 80.0
 dsingle.Geom.concen_region.Lower.Z = 4.0
@@ -114,59 +116,59 @@
 dsingle.Geom.concen_region.Upper.Y = 100.0
 dsingle.Geom.concen_region.Upper.Z = 6.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
-dsingle.Geom.Perm.Names = 'background'
+# -----------------------------------------------------------------------------
+dsingle.Geom.Perm.Names = "background"
 
-dsingle.Geom.background.Perm.Type = 'Constant'
+dsingle.Geom.background.Perm.Type = "Constant"
 dsingle.Geom.background.Perm.Value = 4.0
 
-dsingle.Perm.TensorType = 'TensorByGeom'
+dsingle.Perm.TensorType = "TensorByGeom"
 
-dsingle.Geom.Perm.TensorByGeom.Names = 'background'
+dsingle.Geom.Perm.TensorByGeom.Names = "background"
 
 dsingle.Geom.background.Perm.TensorValX = 1.0
 dsingle.Geom.background.Perm.TensorValY = 1.0
 dsingle.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # specific storage does not figure into the impes (fully sat) case but we still
 # need a key for it
 
-dsingle.SpecificStorage.Type = 'Constant'
-dsingle.SpecificStorage.GeomNames = ''
+dsingle.SpecificStorage.Type = "Constant"
+dsingle.SpecificStorage.GeomNames = ""
 dsingle.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Phase.Names = 'water'
+dsingle.Phase.Names = "water"
 
-dsingle.Phase.water.Density.Type = 'Constant'
+dsingle.Phase.water.Density.Type = "Constant"
 dsingle.Phase.water.Density.Value = 1.0
 
-dsingle.Phase.water.Viscosity.Type = 'Constant'
+dsingle.Phase.water.Viscosity.Type = "Constant"
 dsingle.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
-dsingle.Contaminants.Names = 'tce'
+# -----------------------------------------------------------------------------
+dsingle.Contaminants.Names = "tce"
 dsingle.Contaminants.tce.Degradation.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dsingle.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dsingle.TimingInfo.BaseUnit = 1.0
 dsingle.TimingInfo.StartCount = 0
@@ -174,52 +176,52 @@
 dsingle.TimingInfo.StopTime = 1000.0
 dsingle.TimingInfo.DumpInterval = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.Geom.Porosity.GeomNames = 'background'
+dsingle.Geom.Porosity.GeomNames = "background"
 
-dsingle.Geom.background.Porosity.Type = 'Constant'
+dsingle.Geom.background.Porosity.Type = "Constant"
 dsingle.Geom.background.Porosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
-dsingle.Domain.GeomName = 'domain'
+# -----------------------------------------------------------------------------
+dsingle.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mobility
-#-----------------------------------------------------------------------------
-dsingle.Phase.water.Mobility.Type = 'Constant'
+# -----------------------------------------------------------------------------
+dsingle.Phase.water.Mobility.Type = "Constant"
 dsingle.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
-dsingle.Geom.Retardation.GeomNames = 'background'
-dsingle.Geom.background.tce.Retardation.Type = 'Linear'
+# -----------------------------------------------------------------------------
+dsingle.Geom.Retardation.GeomNames = "background"
+dsingle.Geom.background.tce.Retardation.Type = "Linear"
 dsingle.Geom.background.tce.Retardation.Rate = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
-dsingle.Cycle.Names = 'constant'
-dsingle.Cycle.constant.Names = 'alltime'
+# -----------------------------------------------------------------------------
+dsingle.Cycle.Names = "constant"
+dsingle.Cycle.constant.Names = "alltime"
 dsingle.Cycle.constant.alltime.Length = 1
 dsingle.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
-dsingle.Wells.Names = 'snoopy'
+# -----------------------------------------------------------------------------
+dsingle.Wells.Names = "snoopy"
 
-dsingle.Wells.snoopy.InputType = 'Recirc'
+dsingle.Wells.snoopy.InputType = "Recirc"
 
-dsingle.Wells.snoopy.Cycle = 'constant'
+dsingle.Wells.snoopy.Cycle = "constant"
 
-dsingle.Wells.snoopy.ExtractionType = 'Flux'
-dsingle.Wells.snoopy.InjectionType = 'Flux'
+dsingle.Wells.snoopy.ExtractionType = "Flux"
+dsingle.Wells.snoopy.InjectionType = "Flux"
 
 dsingle.Wells.snoopy.X = 71.0
 dsingle.Wells.snoopy.Y = 90.0
@@ -228,92 +230,92 @@
 dsingle.Wells.snoopy.InjectionZLower = 2.0
 dsingle.Wells.snoopy.InjectionZUpper = 2.0
 
-dsingle.Wells.snoopy.ExtractionMethod = 'Standard'
-dsingle.Wells.snoopy.InjectionMethod = 'Standard'
+dsingle.Wells.snoopy.ExtractionMethod = "Standard"
+dsingle.Wells.snoopy.InjectionMethod = "Standard"
 
 dsingle.Wells.snoopy.alltime.Extraction.Flux.water.Value = 5.0
 dsingle.Wells.snoopy.alltime.Injection.Flux.water.Value = 7.5
 dsingle.Wells.snoopy.alltime.Injection.Concentration.water.tce.Fraction = 0.1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Assigning well with newly assigned interval name
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 dsingle.Wells.snoopy.alltime.Extraction.Flux.water.Value = 5.0
 dsingle.Wells.snoopy.alltime.Injection.Flux.water.Value = 7.5
 dsingle.Wells.snoopy.alltime.Injection.Concentration.water.tce.Fraction = 0.1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
-dsingle.BCPressure.PatchNames = 'left right front back bottom top'
+# -----------------------------------------------------------------------------
+dsingle.BCPressure.PatchNames = "left right front back bottom top"
 
-dsingle.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-dsingle.Patch.left.BCPressure.Cycle = 'constant'
-dsingle.Patch.left.BCPressure.RefGeom = 'domain'
-dsingle.Patch.left.BCPressure.RefPatch = 'bottom'
+dsingle.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+dsingle.Patch.left.BCPressure.Cycle = "constant"
+dsingle.Patch.left.BCPressure.RefGeom = "domain"
+dsingle.Patch.left.BCPressure.RefPatch = "bottom"
 dsingle.Patch.left.BCPressure.alltime.Value = 14.0
 
-dsingle.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-dsingle.Patch.right.BCPressure.Cycle = 'constant'
-dsingle.Patch.right.BCPressure.RefGeom = 'domain'
-dsingle.Patch.right.BCPressure.RefPatch = 'bottom'
+dsingle.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+dsingle.Patch.right.BCPressure.Cycle = "constant"
+dsingle.Patch.right.BCPressure.RefGeom = "domain"
+dsingle.Patch.right.BCPressure.RefPatch = "bottom"
 dsingle.Patch.right.BCPressure.alltime.Value = 9.0
 
-dsingle.Patch.front.BCPressure.Type = 'FluxConst'
-dsingle.Patch.front.BCPressure.Cycle = 'constant'
+dsingle.Patch.front.BCPressure.Type = "FluxConst"
+dsingle.Patch.front.BCPressure.Cycle = "constant"
 dsingle.Patch.front.BCPressure.alltime.Value = 0.0
 
-dsingle.Patch.back.BCPressure.Type = 'FluxConst'
-dsingle.Patch.back.BCPressure.Cycle = 'constant'
+dsingle.Patch.back.BCPressure.Type = "FluxConst"
+dsingle.Patch.back.BCPressure.Cycle = "constant"
 dsingle.Patch.back.BCPressure.alltime.Value = 0.0
 
-dsingle.Patch.bottom.BCPressure.Type = 'FluxConst'
-dsingle.Patch.bottom.BCPressure.Cycle = 'constant'
+dsingle.Patch.bottom.BCPressure.Type = "FluxConst"
+dsingle.Patch.bottom.BCPressure.Cycle = "constant"
 dsingle.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-dsingle.Patch.top.BCPressure.Type = 'FluxConst'
-dsingle.Patch.top.BCPressure.Cycle = 'constant'
+dsingle.Patch.top.BCPressure.Type = "FluxConst"
+dsingle.Patch.top.BCPressure.Cycle = "constant"
 dsingle.Patch.top.BCPressure.alltime.Value = 0.0
 
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # topo slopes do not figure into the impes (fully sat) case but we still
 # need keys for them
 
-dsingle.TopoSlopesX.Type = 'Constant'
-dsingle.TopoSlopesX.GeomNames = 'domain'
+dsingle.TopoSlopesX.Type = "Constant"
+dsingle.TopoSlopesX.GeomNames = "domain"
 
 dsingle.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dsingle.TopoSlopesY.Type = 'Constant'
-dsingle.TopoSlopesY.GeomNames = 'domain'
+dsingle.TopoSlopesY.Type = "Constant"
+dsingle.TopoSlopesY.GeomNames = "domain"
 
 dsingle.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # mannings roughnesses do not figure into the impes (fully sat) case but we still
 # need a key for them
 
-dsingle.Mannings.Type = 'Constant'
-dsingle.Mannings.GeomNames = 'domain'
-dsingle.Mannings.Geom.domain.Value = 0.
+dsingle.Mannings.Type = "Constant"
+dsingle.Mannings.GeomNames = "domain"
+dsingle.Mannings.Geom.domain.Value = 0.0
 
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # dzScale values
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 dsingle.Solver.Nonlinear.VariableDz = True
-dsingle.dzScale.Type = 'nzList'
+dsingle.dzScale.Type = "nzList"
 dsingle.dzScale.nzListNumber = 8
 dsingle.Cell._0.dzScale.Value = 0.5
 dsingle.Cell._1.dzScale.Value = 0.5
@@ -325,16 +327,16 @@
 dsingle.Cell._7.dzScale.Value = 0.5
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dsingle.PhaseSources.water.Type = 'Constant'
-dsingle.PhaseSources.water.GeomNames = 'background'
+dsingle.PhaseSources.water.Type = "Constant"
+dsingle.PhaseSources.water.GeomNames = "background"
 dsingle.PhaseSources.water.Geom.background.Value = 0.0
 
-dsingle.PhaseConcen.water.tce.Type = 'Constant'
-dsingle.PhaseConcen.water.tce.GeomNames = 'concen_region'
+dsingle.PhaseConcen.water.tce.Type = "Constant"
+dsingle.PhaseConcen.water.tce.GeomNames = "concen_region"
 dsingle.PhaseConcen.water.tce.Geom.concen_region.Value = 0.8
 
 
@@ -344,23 +346,25 @@
 dsingle.Solver.WriteSiloConcentration = True
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # The Solver Impes MaxIter default value changed so to get previous
 # results we need to set it back to what it was
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 dsingle.Solver.MaxIter = 5
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 generatedFile, runArg = dsingle.write()
 
 # Prevent regression
-with open(generatedFile) as new, open(get_absolute_path('$PF_SRC/test/correct_output/dsingle.pfidb.ref')) as ref:
-  if new.read() == ref.read():
-    print('Success we have the same file')
-  else:
-    print('Files are different')
-    sys.exit(1)
+with open(generatedFile) as new, open(
+    get_absolute_path("$PF_SRC/test/correct_output/dsingle.pfidb.ref")
+) as ref:
+    if new.read() == ref.read():
+        print("Success we have the same file")
+    else:
+        print("Files are different")
+        sys.exit(1)
diff --git a/test/python/octree-large-domain.py b/test/python/octree-large-domain.py
index 72c195572..e3f1d0f3b 100644
--- a/test/python/octree-large-domain.py
+++ b/test/python/octree-large-domain.py
@@ -2,12 +2,15 @@
 #  This run, as written in this input file, should take
 #  3 nonlinear iterations.
 
+import sys
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
-octree = Run("octree_large_domain", __file__)
+run_name = "octree-large-domain"
+octree = Run(run_name, __file__)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 octree.FileVersion = 4
 
@@ -15,9 +18,9 @@
 octree.Process.Topology.Q = 1
 octree.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 octree.ComputationalGrid.Lower.X = -10.0
 octree.ComputationalGrid.Lower.Y = 10.0
@@ -31,22 +34,24 @@
 octree.ComputationalGrid.NY = 10
 octree.ComputationalGrid.NZ = 10
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-octree.GeomInput.Names = 'domain_input background_input source_region_input concen_region_input'
+octree.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-octree.GeomInput.domain_input.InputType = 'Box'
-octree.GeomInput.domain_input.GeomName = 'domain'
+octree.GeomInput.domain_input.InputType = "Box"
+octree.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 octree.Geom.domain.Lower.X = -10.0
 octree.Geom.domain.Lower.Y = 10.0
@@ -56,18 +61,18 @@
 octree.Geom.domain.Upper.Y = 170.0
 octree.Geom.domain.Upper.Z = 9.0
 
-octree.Geom.domain.Patches = 'left right front back bottom top'
+octree.Geom.domain.Patches = "left right front back bottom top"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-octree.GeomInput.background_input.InputType = 'Box'
-octree.GeomInput.background_input.GeomName = 'background'
+octree.GeomInput.background_input.InputType = "Box"
+octree.GeomInput.background_input.GeomName = "background"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 octree.Geom.background.Lower.X = -99999999.0
 octree.Geom.background.Lower.Y = -99999999.0
@@ -77,16 +82,16 @@
 octree.Geom.background.Upper.Y = 99999999.0
 octree.Geom.background.Upper.Z = 99999999.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-octree.GeomInput.source_region_input.InputType = 'Box'
-octree.GeomInput.source_region_input.GeomName = 'source_region'
+octree.GeomInput.source_region_input.InputType = "Box"
+octree.GeomInput.source_region_input.GeomName = "source_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 octree.Geom.source_region.Lower.X = 65.56
 octree.Geom.source_region.Lower.Y = 79.34
@@ -96,16 +101,16 @@
 octree.Geom.source_region.Upper.Y = 89.99
 octree.Geom.source_region.Upper.Z = 5.5
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-octree.GeomInput.concen_region_input.InputType = 'Box'
-octree.GeomInput.concen_region_input.GeomName = 'concen_region'
+octree.GeomInput.concen_region_input.InputType = "Box"
+octree.GeomInput.concen_region_input.GeomName = "concen_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 octree.Geom.concen_region.Lower.X = 60.0
 octree.Geom.concen_region.Lower.Y = 80.0
@@ -115,230 +120,260 @@
 octree.Geom.concen_region.Upper.Y = 100.0
 octree.Geom.concen_region.Upper.Z = 6.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Geom.Perm.Names = 'background'
+octree.Geom.Perm.Names = "background"
 
-octree.Geom.background.Perm.Type = 'Constant'
+octree.Geom.background.Perm.Type = "Constant"
 octree.Geom.background.Perm.Value = 4.0
 
-octree.Perm.TensorType = 'TensorByGeom'
+octree.Perm.TensorType = "TensorByGeom"
 
-octree.Geom.Perm.TensorByGeom.Names = 'background'
+octree.Geom.Perm.TensorByGeom.Names = "background"
 
 octree.Geom.background.Perm.TensorValX = 1.0
 octree.Geom.background.Perm.TensorValY = 1.0
 octree.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.SpecificStorage.Type = 'Constant'
-octree.SpecificStorage.GeomNames = 'domain'
+octree.SpecificStorage.Type = "Constant"
+octree.SpecificStorage.GeomNames = "domain"
 octree.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Phase.Names = 'water'
+octree.Phase.Names = "water"
 
-octree.Phase.water.Density.Type = 'Constant'
+octree.Phase.water.Density.Type = "Constant"
 octree.Phase.water.Density.Value = 1.0
 
-octree.Phase.water.Viscosity.Type = 'Constant'
+octree.Phase.water.Viscosity.Type = "Constant"
 octree.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Contaminants.Names = ''
+octree.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Geom.Retardation.GeomNames = ''
+octree.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 octree.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 octree.TimingInfo.BaseUnit = 1.0
 octree.TimingInfo.StartCount = 0
 octree.TimingInfo.StartTime = 0.0
 octree.TimingInfo.StopTime = 0.010
 octree.TimingInfo.DumpInterval = -1
-octree.TimeStep.Type = 'Constant'
+octree.TimeStep.Type = "Constant"
 octree.TimeStep.Value = 0.001
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Geom.Porosity.GeomNames = 'background'
+octree.Geom.Porosity.GeomNames = "background"
 
-octree.Geom.background.Porosity.Type = 'Constant'
+octree.Geom.background.Porosity.Type = "Constant"
 octree.Geom.background.Porosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Domain.GeomName = 'domain'
+octree.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Phase.RelPerm.Type = 'VanGenuchten'
-octree.Phase.RelPerm.GeomNames = 'domain'
+octree.Phase.RelPerm.Type = "VanGenuchten"
+octree.Phase.RelPerm.GeomNames = "domain"
 octree.Geom.domain.RelPerm.Alpha = 0.005
 octree.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-octree.Phase.Saturation.Type = 'VanGenuchten'
-octree.Phase.Saturation.GeomNames = 'domain'
+octree.Phase.Saturation.Type = "VanGenuchten"
+octree.Phase.Saturation.GeomNames = "domain"
 octree.Geom.domain.Saturation.Alpha = 0.005
 octree.Geom.domain.Saturation.N = 2.0
 octree.Geom.domain.Saturation.SRes = 0.2
 octree.Geom.domain.Saturation.SSat = 0.99
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Wells.Names = ''
+octree.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Cycle.Names = 'constant'
-octree.Cycle.constant.Names = 'alltime'
+octree.Cycle.Names = "constant"
+octree.Cycle.constant.Names = "alltime"
 octree.Cycle.constant.alltime.Length = 1
 octree.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.BCPressure.PatchNames = 'left right front back bottom top'
+octree.BCPressure.PatchNames = "left right front back bottom top"
 
-octree.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-octree.Patch.left.BCPressure.Cycle = 'constant'
-octree.Patch.left.BCPressure.RefGeom = 'domain'
-octree.Patch.left.BCPressure.RefPatch = 'bottom'
+octree.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+octree.Patch.left.BCPressure.Cycle = "constant"
+octree.Patch.left.BCPressure.RefGeom = "domain"
+octree.Patch.left.BCPressure.RefPatch = "bottom"
 octree.Patch.left.BCPressure.alltime.Value = 5.0
 
-octree.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-octree.Patch.right.BCPressure.Cycle = 'constant'
-octree.Patch.right.BCPressure.RefGeom = 'domain'
-octree.Patch.right.BCPressure.RefPatch = 'bottom'
+octree.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+octree.Patch.right.BCPressure.Cycle = "constant"
+octree.Patch.right.BCPressure.RefGeom = "domain"
+octree.Patch.right.BCPressure.RefPatch = "bottom"
 octree.Patch.right.BCPressure.alltime.Value = 3.0
 
-octree.Patch.front.BCPressure.Type = 'FluxConst'
-octree.Patch.front.BCPressure.Cycle = 'constant'
+octree.Patch.front.BCPressure.Type = "FluxConst"
+octree.Patch.front.BCPressure.Cycle = "constant"
 octree.Patch.front.BCPressure.alltime.Value = 0.0
 
-octree.Patch.back.BCPressure.Type = 'FluxConst'
-octree.Patch.back.BCPressure.Cycle = 'constant'
+octree.Patch.back.BCPressure.Type = "FluxConst"
+octree.Patch.back.BCPressure.Cycle = "constant"
 octree.Patch.back.BCPressure.alltime.Value = 0.0
 
-octree.Patch.bottom.BCPressure.Type = 'FluxConst'
-octree.Patch.bottom.BCPressure.Cycle = 'constant'
+octree.Patch.bottom.BCPressure.Type = "FluxConst"
+octree.Patch.bottom.BCPressure.Cycle = "constant"
 octree.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-octree.Patch.top.BCPressure.Type = 'FluxConst'
-octree.Patch.top.BCPressure.Cycle = 'constant'
+octree.Patch.top.BCPressure.Type = "FluxConst"
+octree.Patch.top.BCPressure.Cycle = "constant"
 octree.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-octree.TopoSlopesX.Type = 'Constant'
-octree.TopoSlopesX.GeomNames = 'domain'
+octree.TopoSlopesX.Type = "Constant"
+octree.TopoSlopesX.GeomNames = "domain"
 
 octree.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-octree.TopoSlopesY.Type = 'Constant'
-octree.TopoSlopesY.GeomNames = 'domain'
+octree.TopoSlopesY.Type = "Constant"
+octree.TopoSlopesY.GeomNames = "domain"
 
 octree.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-octree.Mannings.Type = 'Constant'
-octree.Mannings.GeomNames = 'domain'
-octree.Mannings.Geom.domain.Value = 0.
+octree.Mannings.Type = "Constant"
+octree.Mannings.GeomNames = "domain"
+octree.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-octree.ICPressure.Type = 'HydroStaticPatch'
-octree.ICPressure.GeomNames = 'domain'
+octree.ICPressure.Type = "HydroStaticPatch"
+octree.ICPressure.GeomNames = "domain"
 octree.Geom.domain.ICPressure.Value = 3.0
-octree.Geom.domain.ICPressure.RefGeom = 'domain'
-octree.Geom.domain.ICPressure.RefPatch = 'bottom'
+octree.Geom.domain.ICPressure.RefGeom = "domain"
+octree.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.PhaseSources.water.Type = 'Constant'
-octree.PhaseSources.water.GeomNames = 'background'
+octree.PhaseSources.water.Type = "Constant"
+octree.PhaseSources.water.GeomNames = "background"
 octree.PhaseSources.water.Geom.background.Value = 0.0
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.KnownSolution = 'NoKnownSolution'
+octree.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Solver = 'Richards'
+octree.Solver = "Richards"
 octree.Solver.MaxIter = 5
 
 octree.Solver.Nonlinear.MaxIter = 10
 octree.Solver.Nonlinear.ResidualTol = 1e-9
-octree.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+octree.Solver.Nonlinear.EtaChoice = "EtaConstant"
 octree.Solver.Nonlinear.EtaValue = 1e-5
 octree.Solver.Nonlinear.UseJacobian = True
 octree.Solver.Nonlinear.DerivativeEpsilon = 1e-2
 
 octree.Solver.Linear.KrylovDimension = 10
 
-octree.Solver.Linear.Preconditioner = 'PFMG'
+octree.Solver.Linear.Preconditioner = "PFMG"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/octree_ld')
-mkdir(dir_name)
-octree.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+new_output_dir_name = get_absolute_path("test_output/octree-large-domain")
+mkdir(new_output_dir_name)
+correct_output_dir_name = get_absolute_path("../correct_output")
+octree.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+        sig_digits=4,
+    ):
+        passed = False
+
+for i in range(6):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+        sig_digits=4,
+    ):
+        passed = False
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/octree-simple.py b/test/python/octree-simple.py
index 539a8aeae..57abea199 100644
--- a/test/python/octree-simple.py
+++ b/test/python/octree-simple.py
@@ -2,12 +2,15 @@
 #  This run, as written in this input file, should take
 #  3 nonlinear iterations.
 
+import sys
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
-octree = Run("octree_simple", __file__)
+run_name = "octree-simple"
+octree = Run(run_name, __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 octree.FileVersion = 4
 
@@ -15,9 +18,9 @@
 octree.Process.Topology.Q = 1
 octree.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 octree.ComputationalGrid.Lower.X = -10.0
 octree.ComputationalGrid.Lower.Y = 10.0
@@ -31,22 +34,24 @@
 octree.ComputationalGrid.NY = 8
 octree.ComputationalGrid.NZ = 8
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-octree.GeomInput.Names = 'domain_input background_input source_region_input concen_region_input'
+octree.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-octree.GeomInput.domain_input.InputType = 'Box'
-octree.GeomInput.domain_input.GeomName = 'domain'
+octree.GeomInput.domain_input.InputType = "Box"
+octree.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 octree.Geom.domain.Lower.X = -10.0
 octree.Geom.domain.Lower.Y = 10.0
@@ -56,18 +61,18 @@
 octree.Geom.domain.Upper.Y = 170.0
 octree.Geom.domain.Upper.Z = 9.0
 
-octree.Geom.domain.Patches = 'left right front back bottom top'
+octree.Geom.domain.Patches = "left right front back bottom top"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-octree.GeomInput.background_input.InputType = 'Box'
-octree.GeomInput.background_input.GeomName = 'background'
+octree.GeomInput.background_input.InputType = "Box"
+octree.GeomInput.background_input.GeomName = "background"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 octree.Geom.background.Lower.X = -99999999.0
 octree.Geom.background.Lower.Y = -99999999.0
@@ -77,16 +82,16 @@
 octree.Geom.background.Upper.Y = 99999999.0
 octree.Geom.background.Upper.Z = 99999999.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-octree.GeomInput.source_region_input.InputType = 'Box'
-octree.GeomInput.source_region_input.GeomName = 'source_region'
+octree.GeomInput.source_region_input.InputType = "Box"
+octree.GeomInput.source_region_input.GeomName = "source_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 octree.Geom.source_region.Lower.X = 65.56
 octree.Geom.source_region.Lower.Y = 79.34
@@ -96,16 +101,16 @@
 octree.Geom.source_region.Upper.Y = 89.99
 octree.Geom.source_region.Upper.Z = 5.5
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-octree.GeomInput.concen_region_input.InputType = 'Box'
-octree.GeomInput.concen_region_input.GeomName = 'concen_region'
+octree.GeomInput.concen_region_input.InputType = "Box"
+octree.GeomInput.concen_region_input.GeomName = "concen_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 octree.Geom.concen_region.Lower.X = 60.0
 octree.Geom.concen_region.Lower.Y = 80.0
@@ -115,227 +120,257 @@
 octree.Geom.concen_region.Upper.Y = 100.0
 octree.Geom.concen_region.Upper.Z = 6.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Geom.Perm.Names = 'background'
+octree.Geom.Perm.Names = "background"
 
-octree.Geom.background.Perm.Type = 'Constant'
+octree.Geom.background.Perm.Type = "Constant"
 octree.Geom.background.Perm.Value = 4.0
 
-octree.Perm.TensorType = 'TensorByGeom'
+octree.Perm.TensorType = "TensorByGeom"
 
-octree.Geom.Perm.TensorByGeom.Names = 'background'
+octree.Geom.Perm.TensorByGeom.Names = "background"
 
 octree.Geom.background.Perm.TensorValX = 1.0
 octree.Geom.background.Perm.TensorValY = 1.0
 octree.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.SpecificStorage.Type = 'Constant'
-octree.SpecificStorage.GeomNames = 'domain'
+octree.SpecificStorage.Type = "Constant"
+octree.SpecificStorage.GeomNames = "domain"
 octree.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Phase.Names = 'water'
+octree.Phase.Names = "water"
 
-octree.Phase.water.Density.Type = 'Constant'
+octree.Phase.water.Density.Type = "Constant"
 octree.Phase.water.Density.Value = 1.0
 
-octree.Phase.water.Viscosity.Type = 'Constant'
+octree.Phase.water.Viscosity.Type = "Constant"
 octree.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Contaminants.Names = ''
+octree.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Geom.Retardation.GeomNames = ''
+octree.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 octree.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 octree.TimingInfo.BaseUnit = 1.0
 octree.TimingInfo.StartCount = 0
 octree.TimingInfo.StartTime = 0.0
 octree.TimingInfo.StopTime = 0.010
 octree.TimingInfo.DumpInterval = -1
-octree.TimeStep.Type = 'Constant'
+octree.TimeStep.Type = "Constant"
 octree.TimeStep.Value = 0.001
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Geom.Porosity.GeomNames = 'background'
+octree.Geom.Porosity.GeomNames = "background"
 
-octree.Geom.background.Porosity.Type = 'Constant'
+octree.Geom.background.Porosity.Type = "Constant"
 octree.Geom.background.Porosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Domain.GeomName = 'domain'
+octree.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Phase.RelPerm.Type = 'VanGenuchten'
-octree.Phase.RelPerm.GeomNames = 'domain'
+octree.Phase.RelPerm.Type = "VanGenuchten"
+octree.Phase.RelPerm.GeomNames = "domain"
 octree.Geom.domain.RelPerm.Alpha = 0.005
 octree.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-octree.Phase.Saturation.Type = 'VanGenuchten'
-octree.Phase.Saturation.GeomNames = 'domain'
+octree.Phase.Saturation.Type = "VanGenuchten"
+octree.Phase.Saturation.GeomNames = "domain"
 octree.Geom.domain.Saturation.Alpha = 0.005
 octree.Geom.domain.Saturation.N = 2.0
 octree.Geom.domain.Saturation.SRes = 0.2
 octree.Geom.domain.Saturation.SSat = 0.99
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Wells.Names = ''
+octree.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Cycle.Names = 'constant'
-octree.Cycle.constant.Names = 'alltime'
+octree.Cycle.Names = "constant"
+octree.Cycle.constant.Names = "alltime"
 octree.Cycle.constant.alltime.Length = 1
 octree.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.BCPressure.PatchNames = 'left right front back bottom top'
+octree.BCPressure.PatchNames = "left right front back bottom top"
 
-octree.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-octree.Patch.left.BCPressure.Cycle = 'constant'
-octree.Patch.left.BCPressure.RefGeom = 'domain'
-octree.Patch.left.BCPressure.RefPatch = 'bottom'
+octree.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+octree.Patch.left.BCPressure.Cycle = "constant"
+octree.Patch.left.BCPressure.RefGeom = "domain"
+octree.Patch.left.BCPressure.RefPatch = "bottom"
 octree.Patch.left.BCPressure.alltime.Value = 5.0
 
-octree.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-octree.Patch.right.BCPressure.Cycle = 'constant'
-octree.Patch.right.BCPressure.RefGeom = 'domain'
-octree.Patch.right.BCPressure.RefPatch = 'bottom'
+octree.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+octree.Patch.right.BCPressure.Cycle = "constant"
+octree.Patch.right.BCPressure.RefGeom = "domain"
+octree.Patch.right.BCPressure.RefPatch = "bottom"
 octree.Patch.right.BCPressure.alltime.Value = 3.0
 
-octree.Patch.front.BCPressure.Type = 'FluxConst'
-octree.Patch.front.BCPressure.Cycle = 'constant'
+octree.Patch.front.BCPressure.Type = "FluxConst"
+octree.Patch.front.BCPressure.Cycle = "constant"
 octree.Patch.front.BCPressure.alltime.Value = 0.0
 
-octree.Patch.back.BCPressure.Type = 'FluxConst'
-octree.Patch.back.BCPressure.Cycle = 'constant'
+octree.Patch.back.BCPressure.Type = "FluxConst"
+octree.Patch.back.BCPressure.Cycle = "constant"
 octree.Patch.back.BCPressure.alltime.Value = 0.0
 
-octree.Patch.bottom.BCPressure.Type = 'FluxConst'
-octree.Patch.bottom.BCPressure.Cycle = 'constant'
+octree.Patch.bottom.BCPressure.Type = "FluxConst"
+octree.Patch.bottom.BCPressure.Cycle = "constant"
 octree.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-octree.Patch.top.BCPressure.Type = 'FluxConst'
-octree.Patch.top.BCPressure.Cycle = 'constant'
+octree.Patch.top.BCPressure.Type = "FluxConst"
+octree.Patch.top.BCPressure.Cycle = "constant"
 octree.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-octree.TopoSlopesX.Type = 'Constant'
-octree.TopoSlopesX.GeomNames = 'domain'
+octree.TopoSlopesX.Type = "Constant"
+octree.TopoSlopesX.GeomNames = "domain"
 octree.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-octree.TopoSlopesY.Type = 'Constant'
-octree.TopoSlopesY.GeomNames = 'domain'
+octree.TopoSlopesY.Type = "Constant"
+octree.TopoSlopesY.GeomNames = "domain"
 octree.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-octree.Mannings.Type = 'Constant'
-octree.Mannings.GeomNames = 'domain'
-octree.Mannings.Geom.domain.Value = 0.
+octree.Mannings.Type = "Constant"
+octree.Mannings.GeomNames = "domain"
+octree.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-octree.ICPressure.Type = 'HydroStaticPatch'
-octree.ICPressure.GeomNames = 'domain'
+octree.ICPressure.Type = "HydroStaticPatch"
+octree.ICPressure.GeomNames = "domain"
 octree.Geom.domain.ICPressure.Value = 3.0
-octree.Geom.domain.ICPressure.RefGeom = 'domain'
-octree.Geom.domain.ICPressure.RefPatch = 'bottom'
+octree.Geom.domain.ICPressure.RefGeom = "domain"
+octree.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.PhaseSources.water.Type = 'Constant'
-octree.PhaseSources.water.GeomNames = 'background'
+octree.PhaseSources.water.Type = "Constant"
+octree.PhaseSources.water.GeomNames = "background"
 octree.PhaseSources.water.Geom.background.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.KnownSolution = 'NoKnownSolution'
+octree.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-octree.Solver = 'Richards'
+octree.Solver = "Richards"
 octree.Solver.MaxIter = 5
 
 octree.Solver.Nonlinear.MaxIter = 10
 octree.Solver.Nonlinear.ResidualTol = 1e-9
-octree.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+octree.Solver.Nonlinear.EtaChoice = "EtaConstant"
 octree.Solver.Nonlinear.EtaValue = 1e-5
 octree.Solver.Nonlinear.UseJacobian = True
 octree.Solver.Nonlinear.DerivativeEpsilon = 1e-2
 
 octree.Solver.Linear.KrylovDimension = 10
 
-octree.Solver.Linear.Preconditioner = 'PFMG'
+octree.Solver.Linear.Preconditioner = "PFMG"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/octree')
-mkdir(dir_name)
-octree.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+new_output_dir_name = get_absolute_path("test_output/octree-simple")
+mkdir(new_output_dir_name)
+correct_output_dir_name = get_absolute_path("../correct_output")
+octree.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+        sig_digits=4,
+    ):
+        passed = False
+
+for i in range(6):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+        sig_digits=4,
+    ):
+        passed = False
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/overland_FlatICP.py b/test/python/overland_FlatICP.py
index 900bb51b3..5417e8874 100644
--- a/test/python/overland_FlatICP.py
+++ b/test/python/overland_FlatICP.py
@@ -1,13 +1,15 @@
-#---------------------------------------------------------------------------------
+# ---------------------------------------------------------------------------------
 #  This runs a simple flat box domain with an initial mound of water in the middle
-#---------------------------------------------------------------------------------
+# ---------------------------------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
 overland = Run("overland_FlatICP", __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.FileVersion = 4
 
@@ -15,9 +17,9 @@
 overland.Process.Topology.Q = 1
 overland.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.ComputationalGrid.Lower.X = 0.0
 overland.ComputationalGrid.Lower.Y = 0.0
@@ -29,23 +31,23 @@
 
 overland.ComputationalGrid.DX = 10.0
 overland.ComputationalGrid.DY = 10.0
-overland.ComputationalGrid.DZ = .05
+overland.ComputationalGrid.DZ = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland.GeomInput.Names = 'domaininput sourceinput'
+overland.GeomInput.Names = "domaininput sourceinput"
 
-overland.GeomInput.domaininput.GeomName = 'domain'
-overland.GeomInput.domaininput.InputType = 'Box'
+overland.GeomInput.domaininput.GeomName = "domain"
+overland.GeomInput.domaininput.InputType = "Box"
 
-overland.GeomInput.sourceinput.GeomName = 'icsource'
-overland.GeomInput.sourceinput.InputType = 'Box'
+overland.GeomInput.sourceinput.GeomName = "icsource"
+overland.GeomInput.sourceinput.InputType = "Box"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.Geom.domain.Lower.X = 0.0
 overland.Geom.domain.Lower.Y = 0.0
@@ -54,7 +56,7 @@
 overland.Geom.domain.Upper.X = 100.0
 overland.Geom.domain.Upper.Y = 100.0
 overland.Geom.domain.Upper.Z = 0.05
-overland.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+overland.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
 overland.Geom.icsource.Lower.X = 40.0
 overland.Geom.icsource.Lower.Y = 40.0
@@ -64,288 +66,440 @@
 overland.Geom.icsource.Upper.Y = 60.0
 overland.Geom.icsource.Upper.Z = 0.05
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Geom.Perm.Names = 'domain'
+overland.Geom.Perm.Names = "domain"
 
 # Values in m/hour
-overland.Geom.domain.Perm.Type = 'Constant'
-overland.Geom.domain.Perm.Value = .000694
+overland.Geom.domain.Perm.Type = "Constant"
+overland.Geom.domain.Perm.Value = 0.000694
 
-overland.Perm.TensorType = 'TensorByGeom'
+overland.Perm.TensorType = "TensorByGeom"
 
-overland.Geom.Perm.TensorByGeom.Names = 'domain'
+overland.Geom.Perm.TensorByGeom.Names = "domain"
 
 overland.Geom.domain.Perm.TensorValX = 1.0
 overland.Geom.domain.Perm.TensorValY = 1.0
 overland.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.SpecificStorage.Type = 'Constant'
-overland.SpecificStorage.GeomNames = 'domain'
+overland.SpecificStorage.Type = "Constant"
+overland.SpecificStorage.GeomNames = "domain"
 overland.Geom.domain.SpecificStorage.Value = 5.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Phase.Names = 'water'
+overland.Phase.Names = "water"
 
-overland.Phase.water.Density.Type = 'Constant'
+overland.Phase.water.Density.Type = "Constant"
 overland.Phase.water.Density.Value = 1.0
 
-overland.Phase.water.Viscosity.Type = 'Constant'
+overland.Phase.water.Viscosity.Type = "Constant"
 overland.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Contaminants.Names = ''
+overland.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Geom.Retardation.GeomNames = ''
+overland.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 overland.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 overland.TimingInfo.BaseUnit = 1.0
 overland.TimingInfo.StartCount = 0
 overland.TimingInfo.StartTime = 0.0
 overland.TimingInfo.StopTime = 300.0
 overland.TimingInfo.DumpInterval = 30.0
-overland.TimeStep.Type = 'Constant'
+overland.TimeStep.Type = "Constant"
 overland.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Geom.Porosity.GeomNames = 'domain'
-overland.Geom.domain.Porosity.Type = 'Constant'
+overland.Geom.Porosity.GeomNames = "domain"
+overland.Geom.domain.Porosity.Type = "Constant"
 overland.Geom.domain.Porosity.Value = 0.001
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Domain.GeomName = 'domain'
+overland.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Phase.RelPerm.Type = 'VanGenuchten'
-overland.Phase.RelPerm.GeomNames = 'domain'
+overland.Phase.RelPerm.Type = "VanGenuchten"
+overland.Phase.RelPerm.GeomNames = "domain"
 
 overland.Geom.domain.RelPerm.Alpha = 1.0
-overland.Geom.domain.RelPerm.N = 2.
+overland.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland.Phase.Saturation.Type = 'VanGenuchten'
-overland.Phase.Saturation.GeomNames = 'domain'
+overland.Phase.Saturation.Type = "VanGenuchten"
+overland.Phase.Saturation.GeomNames = "domain"
 
 overland.Geom.domain.Saturation.Alpha = 1.0
-overland.Geom.domain.Saturation.N = 2.
+overland.Geom.domain.Saturation.N = 2.0
 overland.Geom.domain.Saturation.SRes = 0.2
 overland.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Wells.Names = ''
+overland.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Cycle.Names = 'constant rainrec'
-overland.Cycle.constant.Names = 'alltime'
+overland.Cycle.Names = "constant rainrec"
+overland.Cycle.constant.Names = "alltime"
 overland.Cycle.constant.alltime.Length = 1
 overland.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-overland.Cycle.rainrec.Names = 'rain rec'
+overland.Cycle.rainrec.Names = "rain rec"
 overland.Cycle.rainrec.rain.Length = 200
 overland.Cycle.rainrec.rec.Length = 100
 overland.Cycle.rainrec.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 overland.BCPressure.PatchNames = overland.Geom.domain.Patches
 
-overland.Patch.x_lower.BCPressure.Type = 'FluxConst'
-overland.Patch.x_lower.BCPressure.Cycle = 'constant'
+overland.Patch.x_lower.BCPressure.Type = "FluxConst"
+overland.Patch.x_lower.BCPressure.Cycle = "constant"
 overland.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-overland.Patch.y_lower.BCPressure.Type = 'FluxConst'
-overland.Patch.y_lower.BCPressure.Cycle = 'constant'
+overland.Patch.y_lower.BCPressure.Type = "FluxConst"
+overland.Patch.y_lower.BCPressure.Cycle = "constant"
 overland.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-overland.Patch.z_lower.BCPressure.Type = 'FluxConst'
-overland.Patch.z_lower.BCPressure.Cycle = 'constant'
+overland.Patch.z_lower.BCPressure.Type = "FluxConst"
+overland.Patch.z_lower.BCPressure.Cycle = "constant"
 overland.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-overland.Patch.x_upper.BCPressure.Type = 'FluxConst'
-overland.Patch.x_upper.BCPressure.Cycle = 'constant'
+overland.Patch.x_upper.BCPressure.Type = "FluxConst"
+overland.Patch.x_upper.BCPressure.Cycle = "constant"
 overland.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-overland.Patch.y_upper.BCPressure.Type = 'FluxConst'
-overland.Patch.y_upper.BCPressure.Cycle = 'constant'
+overland.Patch.y_upper.BCPressure.Type = "FluxConst"
+overland.Patch.y_upper.BCPressure.Cycle = "constant"
 overland.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
 ## overland flow boundary condition with very heavy rainfall then slight ET
-overland.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-overland.Patch.z_upper.BCPressure.Cycle = 'rainrec'
+overland.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+overland.Patch.z_upper.BCPressure.Cycle = "rainrec"
 overland.Patch.z_upper.BCPressure.rain.Value = 0.0
 overland.Patch.z_upper.BCPressure.rec.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland.TopoSlopesX.Type = 'Constant'
-overland.TopoSlopesX.GeomNames = 'domain'
+overland.TopoSlopesX.Type = "Constant"
+overland.TopoSlopesX.GeomNames = "domain"
 overland.TopoSlopesX.Geom.domain.Value = 0.00
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland.TopoSlopesY.Type = 'Constant'
-overland.TopoSlopesY.GeomNames = 'domain'
+overland.TopoSlopesY.Type = "Constant"
+overland.TopoSlopesY.GeomNames = "domain"
 overland.TopoSlopesY.Geom.domain.Value = 0.00
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland.Mannings.Type = 'Constant'
-overland.Mannings.GeomNames = 'domain'
+overland.Mannings.Type = "Constant"
+overland.Mannings.GeomNames = "domain"
 overland.Mannings.Geom.domain.Value = 0.0003312
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.PhaseSources.water.Type = 'Constant'
-overland.PhaseSources.water.GeomNames = 'domain'
+overland.PhaseSources.water.Type = "Constant"
+overland.PhaseSources.water.GeomNames = "domain"
 overland.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.KnownSolution = 'NoKnownSolution'
+overland.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Solver = 'Richards'
+overland.Solver = "Richards"
 overland.Solver.MaxIter = 30000
 
 overland.Solver.Nonlinear.MaxIter = 300
 overland.Solver.Nonlinear.ResidualTol = 1e-8
-overland.Solver.Nonlinear.EtaChoice = 'Walker1'
-overland.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+overland.Solver.Nonlinear.EtaChoice = "Walker1"
+overland.Solver.Nonlinear.EtaChoice = "EtaConstant"
 overland.Solver.Nonlinear.EtaValue = 0.001
 overland.Solver.Nonlinear.UseJacobian = False
 overland.Solver.Nonlinear.DerivativeEpsilon = 1e-12
 overland.Solver.Nonlinear.StepTol = 1e-30
-overland.Solver.Nonlinear.Globalization = 'LineSearch'
+overland.Solver.Nonlinear.Globalization = "LineSearch"
 overland.Solver.Linear.KrylovDimension = 20
 overland.Solver.Linear.MaxRestart = 2
-overland.Solver.OverlandDiffusive.Epsilon = 1E-5
+overland.Solver.OverlandDiffusive.Epsilon = 1e-5
 
-overland.Solver.Linear.Preconditioner = 'PFMG'
+overland.Solver.Linear.Preconditioner = "PFMG"
 overland.Solver.PrintSubsurf = False
-overland.Solver.Drop = 1E-20
-overland.Solver.AbsTol = 1E-12
+overland.Solver.Drop = 1e-20
+overland.Solver.AbsTol = 1e-12
 
 overland.Solver.WriteSiloSubsurfData = False
 overland.Solver.WriteSiloPressure = False
 overland.Solver.WriteSiloSaturation = False
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland.ICPressure.Type = 'HydroStaticPatch'
-overland.ICPressure.GeomNames = 'domain icsource'
+overland.ICPressure.Type = "HydroStaticPatch"
+overland.ICPressure.GeomNames = "domain icsource"
 
 overland.Geom.domain.ICPressure.Value = 0.0
-overland.Geom.domain.ICPressure.RefGeom = 'domain'
-overland.Geom.domain.ICPressure.RefPatch = 'z_upper'
+overland.Geom.domain.ICPressure.RefGeom = "domain"
+overland.Geom.domain.ICPressure.RefPatch = "z_upper"
 
 overland.Geom.icsource.ICPressure.Value = 0.1
-overland.Geom.icsource.ICPressure.RefGeom = 'domain'
-overland.Geom.icsource.ICPressure.RefPatch = 'z_upper'
+overland.Geom.icsource.ICPressure.RefGeom = "domain"
+overland.Geom.icsource.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+
+runcheck = 1
+correct_output_dir_name = get_absolute_path("../correct_output")
+
+# -----------------------------------------------------------------------------
 # original approach from K&M AWR 2006
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
+overland.Patch.z_upper.BCPressure.Type = "OverlandFlow"
 overland.Solver.Nonlinear.UseJacobian = False
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'PFSymmetric'
-
-#-----------------------------------------------------------------------------
+overland.Solver.Linear.Preconditioner.PCMatrixType = "PFSymmetric"
+
+run_name = "FlatICP_Overland"
+overland.set_name(run_name)
+print(run_name)
+new_output_dir_name = get_absolute_path("test_output/" + run_name)
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if runcheck == 1:
+    passed = True
+    for i in range(11):
+        timestep = str(i).rjust(5, "0")
+        filename = f"/{run_name}.out.press.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Pressure for timestep {timestep}",
+        ):
+            passed = False
+        filename = f"/{run_name}.out.satur.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Saturation for timestep {timestep}",
+        ):
+            passed = False
+
+    if passed:
+        print(f"{run_name} : PASSED")
+    else:
+        print(f"{run_name} : FAILED")
+        sys.exit(1)
+
+
+# -----------------------------------------------------------------------------
 # New kinematic formulation - this should exactly match the original formulation
 #  for this flat test case
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Patch.z_upper.BCPressure.Type = 'OverlandKinematic'
+overland.Patch.z_upper.BCPressure.Type = "OverlandKinematic"
 overland.Solver.Nonlinear.UseJacobian = False
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'PFSymmetric'
-
-#-----------------------------------------------------------------------------
+overland.Solver.Linear.Preconditioner.PCMatrixType = "PFSymmetric"
+
+run_name = "FlatICP_OverlandKin"
+overland.set_name(run_name)
+print(f"Running {run_name}")
+new_output_dir_name = get_absolute_path("test_output/" + run_name)
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if runcheck == 1:
+    passed = True
+    for i in range(11):
+        timestep = str(i).rjust(5, "0")
+        filename = f"/{run_name}.out.press.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Pressure for timestep {timestep}",
+        ):
+            passed = False
+        filename = f"/{run_name}.out.satur.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Saturation for timestep {timestep}",
+        ):
+            passed = False
+
+    if passed:
+        print(f"{run_name} : PASSED")
+    else:
+        print(f"{run_name} : FAILED")
+        sys.exit(1)
+
+
+# -----------------------------------------------------------------------------
 # Diffusive formulation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-#run with Jacobian False
-overland.Patch.z_upper.BCPressure.Type = 'OverlandDiffusive'
+# run with Jacobian False
+overland.Patch.z_upper.BCPressure.Type = "OverlandDiffusive"
 overland.Solver.Nonlinear.UseJacobian = False
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'PFSymmetric'
+overland.Solver.Linear.Preconditioner.PCMatrixType = "PFSymmetric"
+
+run_name = "FlatICP_OverlandDif"
+overland.set_name(run_name)
+print(f"Running {run_name}")
+new_output_dir_name = get_absolute_path("test_output/" + run_name)
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if runcheck == 1:
+    passed = True
+    for i in range(11):
+        timestep = str(i).rjust(5, "0")
+        filename = f"/{run_name}.out.press.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Pressure for timestep {timestep}",
+        ):
+            passed = False
+        filename = f"/{run_name}.out.satur.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Saturation for timestep {timestep}",
+        ):
+            passed = False
+
+    if passed:
+        print(f"{run_name} : PASSED")
+    else:
+        print(f"{run_name} : FAILED")
+        sys.exit(1)
+
 
 # run with analytical jacobian
-overland.Patch.z_upper.BCPressure.Type = 'OverlandDiffusive'
+overland.Patch.z_upper.BCPressure.Type = "OverlandDiffusive"
 overland.Solver.Nonlinear.UseJacobian = True
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'PFSymmetric'
+overland.Solver.Linear.Preconditioner.PCMatrixType = "PFSymmetric"
+
+print(f"Running {run_name} Jacobian True")
+new_output_dir_name = get_absolute_path("test_output/" + f"{run_name}_jacobian_true")
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if runcheck == 1:
+    passed = True
+    for i in range(11):
+        timestep = str(i).rjust(5, "0")
+        filename = f"/{run_name}.out.press.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Pressure for timestep {timestep}",
+        ):
+            passed = False
+        filename = f"/{run_name}.out.satur.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Saturation for timestep {timestep}",
+        ):
+            passed = False
+
+    if passed:
+        print(f"{run_name} : PASSED")
+    else:
+        print(f"{run_name} : FAILED")
+        sys.exit(1)
+
 
 # run with analytical jacobian and nonsymmetric preconditioner
-overland.Patch.z_upper.BCPressure.Type = 'OverlandDiffusive'
+overland.Patch.z_upper.BCPressure.Type = "OverlandDiffusive"
 overland.Solver.Nonlinear.UseJacobian = True
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'FullJacobian'
-
-#-----------------------------------------------------------------------------
-# Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/o_ficp')
-mkdir(dir_name)
-overland.run(working_directory=dir_name)
-
+overland.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
+
+print(f"Running {run_name} Jacobian True Nonsymmetric Preconditioner")
+new_output_dir_name = get_absolute_path(
+    "test_output/" + f"{run_name}_jacobian_true_nonsymmetric_preconditioner"
+)
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if runcheck == 1:
+    passed = True
+    for i in range(11):
+        timestep = str(i).rjust(5, "0")
+        filename = f"/{run_name}.out.press.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Pressure for timestep {timestep}",
+        ):
+            passed = False
+        filename = f"/{run_name}.out.satur.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Saturation for timestep {timestep}",
+        ):
+            passed = False
+
+    if passed:
+        print(f"{run_name} : PASSED")
+    else:
+        print(f"{run_name} : FAILED")
+        sys.exit(1)
diff --git a/test/python/overland_slopingslab_DWE.py b/test/python/overland_slopingslab_DWE.py
index af057b6e7..db4a1b560 100644
--- a/test/python/overland_slopingslab_DWE.py
+++ b/test/python/overland_slopingslab_DWE.py
@@ -1,15 +1,17 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  Testing overland flow diffusive wave
 # Running a parking lot sloping slab pointed in 8 directions
 # With diffusive BC options
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
 overland = Run("overland_slopingslab_DWE", __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.FileVersion = 4
 
@@ -17,9 +19,9 @@
 overland.Process.Topology.Q = 1
 overland.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.ComputationalGrid.Lower.X = 0.0
 overland.ComputationalGrid.Lower.Y = 0.0
@@ -31,19 +33,19 @@
 
 overland.ComputationalGrid.DX = 10.0
 overland.ComputationalGrid.DY = 10.0
-overland.ComputationalGrid.DZ = .05
+overland.ComputationalGrid.DZ = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland.GeomInput.Names = 'domaininput'
-overland.GeomInput.domaininput.GeomName = 'domain'
-overland.GeomInput.domaininput.InputType = 'Box'
+overland.GeomInput.Names = "domaininput"
+overland.GeomInput.domaininput.GeomName = "domain"
+overland.GeomInput.domaininput.InputType = "Box"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.Geom.domain.Lower.X = 0.0
 overland.Geom.domain.Lower.Y = 0.0
@@ -52,213 +54,213 @@
 overland.Geom.domain.Upper.X = 50.0
 overland.Geom.domain.Upper.Y = 50.0
 overland.Geom.domain.Upper.Z = 0.05
-overland.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+overland.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Geom.Perm.Names = 'domain'
-overland.Geom.domain.Perm.Type = 'Constant'
+overland.Geom.Perm.Names = "domain"
+overland.Geom.domain.Perm.Type = "Constant"
 overland.Geom.domain.Perm.Value = 0.0000001
 
-overland.Perm.TensorType = 'TensorByGeom'
+overland.Perm.TensorType = "TensorByGeom"
 
-overland.Geom.Perm.TensorByGeom.Names = 'domain'
+overland.Geom.Perm.TensorByGeom.Names = "domain"
 
 overland.Geom.domain.Perm.TensorValX = 1.0
 overland.Geom.domain.Perm.TensorValY = 1.0
 overland.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.SpecificStorage.Type = 'Constant'
-overland.SpecificStorage.GeomNames = 'domain'
+overland.SpecificStorage.Type = "Constant"
+overland.SpecificStorage.GeomNames = "domain"
 overland.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Phase.Names = 'water'
+overland.Phase.Names = "water"
 
-overland.Phase.water.Density.Type = 'Constant'
+overland.Phase.water.Density.Type = "Constant"
 overland.Phase.water.Density.Value = 1.0
 
-overland.Phase.water.Viscosity.Type = 'Constant'
+overland.Phase.water.Viscosity.Type = "Constant"
 overland.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Contaminants.Names = ''
+overland.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Geom.Retardation.GeomNames = ''
+overland.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 overland.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 overland.TimingInfo.BaseUnit = 0.05
 overland.TimingInfo.StartCount = 0
 overland.TimingInfo.StartTime = 0.0
 overland.TimingInfo.StopTime = 1.0
 overland.TimingInfo.DumpInterval = -2
-overland.TimeStep.Type = 'Constant'
+overland.TimeStep.Type = "Constant"
 overland.TimeStep.Value = 0.05
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Geom.Porosity.GeomNames = 'domain'
+overland.Geom.Porosity.GeomNames = "domain"
 
-overland.Geom.domain.Porosity.Type = 'Constant'
+overland.Geom.domain.Porosity.Type = "Constant"
 overland.Geom.domain.Porosity.Value = 0.01
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Domain.GeomName = 'domain'
+overland.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Phase.RelPerm.Type = 'VanGenuchten'
-overland.Phase.RelPerm.GeomNames = 'domain'
+overland.Phase.RelPerm.Type = "VanGenuchten"
+overland.Phase.RelPerm.GeomNames = "domain"
 
 overland.Geom.domain.RelPerm.Alpha = 6.0
-overland.Geom.domain.RelPerm.N = 2.
+overland.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland.Phase.Saturation.Type = 'VanGenuchten'
-overland.Phase.Saturation.GeomNames = 'domain'
+overland.Phase.Saturation.Type = "VanGenuchten"
+overland.Phase.Saturation.GeomNames = "domain"
 
 overland.Geom.domain.Saturation.Alpha = 6.0
-overland.Geom.domain.Saturation.N = 2.
+overland.Geom.domain.Saturation.N = 2.0
 overland.Geom.domain.Saturation.SRes = 0.2
 overland.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Wells.Names = ''
+overland.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Cycle.Names = 'constant rainrec'
-overland.Cycle.constant.Names = 'alltime'
+overland.Cycle.Names = "constant rainrec"
+overland.Cycle.constant.Names = "alltime"
 overland.Cycle.constant.alltime.Length = 1
 overland.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-overland.Cycle.rainrec.Names = 'rain rec'
+overland.Cycle.rainrec.Names = "rain rec"
 overland.Cycle.rainrec.rain.Length = 2
 overland.Cycle.rainrec.rec.Length = 300
 overland.Cycle.rainrec.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 overland.BCPressure.PatchNames = overland.Geom.domain.Patches
 
-overland.Patch.x_lower.BCPressure.Type = 'FluxConst'
-overland.Patch.x_lower.BCPressure.Cycle = 'constant'
+overland.Patch.x_lower.BCPressure.Type = "FluxConst"
+overland.Patch.x_lower.BCPressure.Cycle = "constant"
 overland.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-overland.Patch.y_lower.BCPressure.Type = 'FluxConst'
-overland.Patch.y_lower.BCPressure.Cycle = 'constant'
+overland.Patch.y_lower.BCPressure.Type = "FluxConst"
+overland.Patch.y_lower.BCPressure.Cycle = "constant"
 overland.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-overland.Patch.z_lower.BCPressure.Type = 'FluxConst'
-overland.Patch.z_lower.BCPressure.Cycle = 'constant'
+overland.Patch.z_lower.BCPressure.Type = "FluxConst"
+overland.Patch.z_lower.BCPressure.Cycle = "constant"
 overland.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-overland.Patch.x_upper.BCPressure.Type = 'FluxConst'
-overland.Patch.x_upper.BCPressure.Cycle = 'constant'
+overland.Patch.x_upper.BCPressure.Type = "FluxConst"
+overland.Patch.x_upper.BCPressure.Cycle = "constant"
 overland.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-overland.Patch.y_upper.BCPressure.Type = 'FluxConst'
-overland.Patch.y_upper.BCPressure.Cycle = 'constant'
+overland.Patch.y_upper.BCPressure.Type = "FluxConst"
+overland.Patch.y_upper.BCPressure.Cycle = "constant"
 overland.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
 ## overland flow boundary condition with very heavy rainfall then slight ET
-overland.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-overland.Patch.z_upper.BCPressure.Cycle = 'rainrec'
+overland.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+overland.Patch.z_upper.BCPressure.Cycle = "rainrec"
 overland.Patch.z_upper.BCPressure.rain.Value = -0.01
 overland.Patch.z_upper.BCPressure.rec.Value = 0.0000
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland.Mannings.Type = 'Constant'
-overland.Mannings.GeomNames = 'domain'
-overland.Mannings.Geom.domain.Value = 3.e-6
+overland.Mannings.Type = "Constant"
+overland.Mannings.GeomNames = "domain"
+overland.Mannings.Geom.domain.Value = 3.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.PhaseSources.water.Type = 'Constant'
-overland.PhaseSources.water.GeomNames = 'domain'
+overland.PhaseSources.water.Type = "Constant"
+overland.PhaseSources.water.GeomNames = "domain"
 overland.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.KnownSolution = 'NoKnownSolution'
+overland.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Solver = 'Richards'
+overland.Solver = "Richards"
 overland.Solver.MaxIter = 2500
 
 overland.Solver.Nonlinear.MaxIter = 50
 overland.Solver.Nonlinear.ResidualTol = 1e-9
-overland.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+overland.Solver.Nonlinear.EtaChoice = "EtaConstant"
 overland.Solver.Nonlinear.EtaValue = 0.01
 overland.Solver.Nonlinear.UseJacobian = False
 
 overland.Solver.Nonlinear.DerivativeEpsilon = 1e-15
 overland.Solver.Nonlinear.StepTol = 1e-20
-overland.Solver.Nonlinear.Globalization = 'LineSearch'
+overland.Solver.Nonlinear.Globalization = "LineSearch"
 overland.Solver.Linear.KrylovDimension = 20
 overland.Solver.Linear.MaxRestart = 2
 
-overland.Solver.Linear.Preconditioner = 'PFMG'
+overland.Solver.Linear.Preconditioner = "PFMG"
 overland.Solver.PrintSubsurf = False
-overland.Solver.Drop = 1E-20
-overland.Solver.AbsTol = 1E-10
+overland.Solver.Drop = 1e-20
+overland.Solver.AbsTol = 1e-10
 
-overland.Solver.OverlandDiffusive.Epsilon = 1E-5
+overland.Solver.OverlandDiffusive.Epsilon = 1e-5
 
 overland.Solver.WriteSiloSubsurfData = False
 overland.Solver.WriteSiloPressure = False
@@ -267,173 +269,196 @@
 overland.Solver.WriteSiloSaturation = False
 overland.Solver.WriteSiloConcentration = False
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be at the bottom of the domain, the top layer is initially dry
-overland.ICPressure.Type = 'HydroStaticPatch'
-overland.ICPressure.GeomNames = 'domain'
+overland.ICPressure.Type = "HydroStaticPatch"
+overland.ICPressure.GeomNames = "domain"
 overland.Geom.domain.ICPressure.Value = -3.0
 
-overland.Geom.domain.ICPressure.RefGeom = 'domain'
-overland.Geom.domain.ICPressure.RefPatch = 'z_upper'
+overland.Geom.domain.ICPressure.RefGeom = "domain"
+overland.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
 # Running all 8 direction combinations with the upwind formulation on and off (i.e. 16 total)
 # Commented lines are from original TCL test - will need to convert to Python if running with
 # Python pftools
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-#set runcheck to 1 if you want to run the pass fail tests
-# runcheck = 1
-# source pftest.tcl
-# first = 1
+# set runcheck to 1 if you want to run the pass fail tests
+runcheck = 1
+first = 1
+correct_output_dir_name = get_absolute_path("../correct_output")
 
 ###############################
 # Looping over slab configurations
 ###############################
-# foreach xslope [list 0.01 -0.01] yslope [list 0.01 -0.01] name [list posxposy negxnegy] {
-#   puts "$xslope $yslope $name"
-
-#   #### Set the slopes
-#   pfset TopoSlopesX.Type "Constant"
-#   pfset TopoSlopesX.GeomNames "domain"
-#   pfset TopoSlopesX.Geom.domain.Value $xslope
-
-#   pfset TopoSlopesY.Type "Constant"
-#   pfset TopoSlopesY.GeomNames "domain"
-#   pfset TopoSlopesY.Geom.domain.Value $yslope
-
-#    #new BC
-#    pfset Patch.z-upper.BCPressure.Type		      OverlandDiffusive
-#    #pfset Solver.Nonlinear.UseJacobian                       True
-#    #pfset Solver.Linear.Preconditioner.PCMatrixType         FullJacobian
-
-#    set runname Slab.$name.OverlandDif
-#    if $first==1 {
-#      puts $runname
-#      set first 0
-#    } else {
-#      puts "Running $runname OverlandDiffusive"
-#    }
-#    pfrun $runname
-#    pfundist $runname
-#    if $runcheck==1 {
-#      set passed 1
-#      foreach i "00000 00001 00002 00003 00004 00005 00006 00007 00008 00009 00010" {
-#        if ![pftestFile $runname.out.press.$i.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
-#          set passed 0
-#        }
-#        if ![pftestFile  $runname.out.satur.$i.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
-#          set passed 0
-#        }
-#      }
-#      if $passed {
-#        puts "$runname : PASSED"
-#      } {
-#        puts "$runname : FAILED"
-#      }
-#    }
-
-# }
-
-#set to 1 if you want to run the pass fail tests
-# runcheck = 1
-# source pftest.tcl
+x_slopes = [0.01, -0.01]
+y_slopes = [0.01, -0.01]
+names = ["posxposy", "negxnegy"]
+
+for x_slope, y_slope, name in zip(x_slopes, y_slopes, names):
+    print(f"{x_slope} {y_slope} {name}")
+    overland.TopoSlopesX.Type = "Constant"
+    overland.TopoSlopesX.GeomNames = "domain"
+    overland.TopoSlopesX.Geom.domain.Value = x_slope
+
+    overland.TopoSlopesY.Type = "Constant"
+    overland.TopoSlopesY.GeomNames = "domain"
+    overland.TopoSlopesY.Geom.domain.Value = y_slope
+
+    overland.Patch.z_upper.BCPressure.Type = "OverlandDiffusive"
+
+    run_name = f"Slab.{name}.OverlandDif"
+    overland.set_name(run_name)
+    if first == 1:
+        print(overland.get_name())
+        first = 0
+    else:
+        print(f"Running {run_name} OverlandDiffusive")
+    new_output_dir_name = get_absolute_path("test_output/" + run_name)
+    mkdir(new_output_dir_name)
+    overland.run(working_directory=new_output_dir_name)
+    if runcheck == 1:
+        passed = True
+        for i in range(11):
+            timestep = str(i).rjust(5, "0")
+            filename = f"/{run_name}.out.press.{timestep}.pfb"
+            if not pf_test_file(
+                new_output_dir_name + filename,
+                correct_output_dir_name + filename,
+                f"Max difference in Pressure for timestep {timestep}",
+            ):
+                passed = False
+            filename = f"/{run_name}.out.satur.{timestep}.pfb"
+            if not pf_test_file(
+                new_output_dir_name + filename,
+                correct_output_dir_name + filename,
+                f"Max difference in Saturation for timestep {timestep}",
+            ):
+                passed = False
+
+        if passed:
+            print(f"{run_name} : PASSED")
+        else:
+            print(f"{run_name} : FAILED")
+            sys.exit(1)
+
+
+# set to 1 if you want to run the pass fail tests
+runcheck = 1
 
 ###############################
 # Looping over slop configurations
 ###############################
-# foreach xslope [list 0.01 -0.01] yslope [list 0.01 -0.01] name [list posxposy negxnegy] {
-#   puts "$xslope $yslope $name"
+x_slopes = [0.01, -0.01]
+y_slopes = [0.01, -0.01]
+names = ["posxposy", "negxnegy"]
+
+for x_slope, y_slope, name in zip(x_slopes, y_slopes, names):
+    print(f"{x_slope} {y_slope} {name}")
+    overland.TopoSlopesX.Type = "Constant"
+    overland.TopoSlopesX.GeomNames = "domain"
+    overland.TopoSlopesX.Geom.domain.Value = x_slope
+
+    overland.TopoSlopesY.Type = "Constant"
+    overland.TopoSlopesY.GeomNames = "domain"
+    overland.TopoSlopesY.Geom.domain.Value = y_slope
+
+    overland.Patch.z_upper.BCPressure.Type = "OverlandDiffusive"
+    overland.Solver.Nonlinear.UseJacobian = True
+
+    run_name = f"Slab.{name}.OverlandDif"
+    overland.set_name(run_name)
+
+    print(f"Running {run_name} OverlandDiffusive Jacobian True")
+    new_output_dir_name = get_absolute_path("test_output/" + run_name)
+    mkdir(new_output_dir_name)
+    overland.run(working_directory=new_output_dir_name)
+    if runcheck == 1:
+        passed = True
+        for i in range(11):
+            timestep = str(i).rjust(5, "0")
+            filename = f"/{run_name}.out.press.{timestep}.pfb"
+            if not pf_test_file(
+                new_output_dir_name + filename,
+                correct_output_dir_name + filename,
+                f"Max difference in Pressure for timestep {timestep}",
+            ):
+                passed = False
+            filename = f"/{run_name}.out.satur.{timestep}.pfb"
+            if not pf_test_file(
+                new_output_dir_name + filename,
+                correct_output_dir_name + filename,
+                f"Max difference in Saturation for timestep {timestep}",
+            ):
+                passed = False
+
+        if passed:
+            print(f"{run_name} : PASSED")
+        else:
+            print(f"{run_name} : FAILED")
+            sys.exit(1)
+
+
+# set to 1 if you want to run the pass fail tests
+runcheck = 1
 
-#   #### Set the slopes
-overland.TopoSlopesX.Type = "Constant"
-overland.TopoSlopesX.GeomNames = "domain"
-overland.TopoSlopesX.Geom.domain.Value = 0.01
-
-overland.TopoSlopesY.Type = "Constant"
-overland.TopoSlopesY.GeomNames = "domain"
-overland.TopoSlopesY.Geom.domain.Value = 0.01
-
-#   #original approach from K&M AWR 2006
-overland.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-overland.Solver.Nonlinear.UseJacobian = False
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'PFSymmetric'
-
-#    set runname Slab.$name.OverlandDif
-#    puts "Running $runname OverlandDiffusive Jacobian True"
-#    pfrun $runname
-#    pfundist $runname
-#    if $runcheck==1 {
-#      set passed 1
-#      foreach i "00000 00001 00002 00003 00004 00005 00006 00007 00008 00009 00010" {
-#        if ![pftestFile $runname.out.press.$i.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
-#          set passed 0
-#        }
-#        if ![pftestFile  $runname.out.satur.$i.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
-#          set passed 0
-#        }
-#      }
-#      if $passed {
-#        puts "$runname : PASSED"
-#      } {
-#        puts "$runname : FAILED"
-#      }
-#    }
-
-
-# }
-
-#set to 1 if you want to run the pass fail tests
-# runcheck = 1
-# source pftest.tcl
 
 ###############################
 # Looping over slop configurations
 ###############################
-# foreach xslope [list 0.01 -0.01] yslope [list 0.01 -0.01] name [list posxposy negxnegy] {
-#   puts "$xslope $yslope $name"
-
-#   #### Set the slopes
-#   pfset TopoSlopesX.Type "Constant"
-#   pfset TopoSlopesX.GeomNames "domain"
-#   pfset TopoSlopesX.Geom.domain.Value $xslope
-
-#   pfset TopoSlopesY.Type "Constant"
-#   pfset TopoSlopesY.GeomNames "domain"
-#   pfset TopoSlopesY.Geom.domain.Value $yslope
-
-#    #new BC
-#    pfset Patch.z-upper.BCPressure.Type		      OverlandDiffusive
-#    pfset Solver.Nonlinear.UseJacobian                       True
-#    pfset Solver.Linear.Preconditioner.PCMatrixType         FullJacobian
-
-#    set runname Slab.$name.OverlandDif
-#    puts "Running $runname OverlandDiffusive Jacobian True Nonsymmetric Preconditioner"
-#    pfrun $runname
-#    pfundist $runname
-#    if $runcheck==1 {
-#      set passed 1
-#      foreach i "00000 00001 00002 00003 00004 00005 00006 00007 00008 00009 00010" {
-#        if ![pftestFile $runname.out.press.$i.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
-#          set passed 0
-#        }
-#        if ![pftestFile  $runname.out.satur.$i.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
-#          set passed 0
-#        }
-#      }
-#      if $passed {
-#        puts "$runname : PASSED"
-#      } {
-#        puts "$runname : FAILED"
-#      }
-#    }
-
-# }
-
-dir_name = get_absolute_path('test_output/os_dwe')
-mkdir(dir_name)
-overland.run(working_directory=dir_name)
+x_slopes = [0.01, -0.01]
+y_slopes = [0.01, -0.01]
+names = ["posxposy", "negxnegy"]
+
+for x_slope, y_slope, name in zip(x_slopes, y_slopes, names):
+    print(f"{x_slope} {y_slope} {name}")
+    overland.TopoSlopesX.Type = "Constant"
+    overland.TopoSlopesX.GeomNames = "domain"
+    overland.TopoSlopesX.Geom.domain.Value = x_slope
+
+    overland.TopoSlopesY.Type = "Constant"
+    overland.TopoSlopesY.GeomNames = "domain"
+    overland.TopoSlopesY.Geom.domain.Value = y_slope
+
+    overland.Patch.z_upper.BCPressure.Type = "OverlandDiffusive"
+    overland.Solver.Nonlinear.UseJacobian = True
+    overland.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
+
+    run_name = f"Slab.{name}.OverlandDif"
+    overland.set_name(run_name)
+
+    print(
+        f"Running {run_name} OverlandDiffusive Jacobian True Nonsymmetric Preconditioner"
+    )
+    new_output_dir_name = get_absolute_path("test_output/" + run_name)
+    mkdir(new_output_dir_name)
+    overland.run(working_directory=new_output_dir_name)
+    if runcheck == 1:
+        passed = True
+        for i in range(11):
+            timestep = str(i).rjust(5, "0")
+            filename = f"/{run_name}.out.press.{timestep}.pfb"
+            if not pf_test_file(
+                new_output_dir_name + filename,
+                correct_output_dir_name + filename,
+                f"Max difference in Pressure for timestep {timestep}",
+            ):
+                passed = False
+            filename = f"/{run_name}.out.satur.{timestep}.pfb"
+            if not pf_test_file(
+                new_output_dir_name + filename,
+                correct_output_dir_name + filename,
+                f"Max difference in Saturation for timestep {timestep}",
+            ):
+                passed = False
+
+        if passed:
+            print(f"{run_name} : PASSED")
+        else:
+            print(f"{run_name} : FAILED")
+            sys.exit(1)
diff --git a/test/python/overland_slopingslab_KWE.py b/test/python/overland_slopingslab_KWE.py
index 74ab2f9e2..773dba6e2 100644
--- a/test/python/overland_slopingslab_KWE.py
+++ b/test/python/overland_slopingslab_KWE.py
@@ -1,25 +1,33 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  Testing overland flow
 # Running a parking lot sloping slab pointed in 8 directions
 # With a suite of overlandflow BC options
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
+import sys, argparse
 from parflow import Run
-from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.fs import mkdir, get_absolute_path, rm
+from parflow.tools.compare import pf_test_file
 
 overland = Run("overland_slopingslab_KWE", __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.FileVersion = 4
 
-overland.Process.Topology.P = 1
-overland.Process.Topology.Q = 1
-overland.Process.Topology.R = 1
+parser = argparse.ArgumentParser()
+parser.add_argument("-p", "--p", default=1)
+parser.add_argument("-q", "--q", default=1)
+parser.add_argument("-r", "--r", default=1)
+args = parser.parse_args()
 
-#---------------------------------------------------------
+overland.Process.Topology.P = args.p
+overland.Process.Topology.Q = args.q
+overland.Process.Topology.R = args.r
+
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.ComputationalGrid.Lower.X = 0.0
 overland.ComputationalGrid.Lower.Y = 0.0
@@ -31,19 +39,19 @@
 
 overland.ComputationalGrid.DX = 10.0
 overland.ComputationalGrid.DY = 10.0
-overland.ComputationalGrid.DZ = .05
+overland.ComputationalGrid.DZ = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland.GeomInput.Names = 'domaininput'
-overland.GeomInput.domaininput.GeomName = 'domain'
-overland.GeomInput.domaininput.InputType = 'Box'
+overland.GeomInput.Names = "domaininput"
+overland.GeomInput.domaininput.GeomName = "domain"
+overland.GeomInput.domaininput.InputType = "Box"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.Geom.domain.Lower.X = 0.0
 overland.Geom.domain.Lower.Y = 0.0
@@ -52,212 +60,212 @@
 overland.Geom.domain.Upper.X = 50.0
 overland.Geom.domain.Upper.Y = 50.0
 overland.Geom.domain.Upper.Z = 0.05
-overland.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+overland.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Geom.Perm.Names = 'domain'
-overland.Geom.domain.Perm.Type = 'Constant'
+overland.Geom.Perm.Names = "domain"
+overland.Geom.domain.Perm.Type = "Constant"
 overland.Geom.domain.Perm.Value = 0.0000001
 
-overland.Perm.TensorType = 'TensorByGeom'
+overland.Perm.TensorType = "TensorByGeom"
 
-overland.Geom.Perm.TensorByGeom.Names = 'domain'
+overland.Geom.Perm.TensorByGeom.Names = "domain"
 
 overland.Geom.domain.Perm.TensorValX = 1.0
 overland.Geom.domain.Perm.TensorValY = 1.0
 overland.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.SpecificStorage.Type = 'Constant'
-overland.SpecificStorage.GeomNames = 'domain'
+overland.SpecificStorage.Type = "Constant"
+overland.SpecificStorage.GeomNames = "domain"
 overland.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Phase.Names = 'water'
+overland.Phase.Names = "water"
 
-overland.Phase.water.Density.Type = 'Constant'
+overland.Phase.water.Density.Type = "Constant"
 overland.Phase.water.Density.Value = 1.0
 
-overland.Phase.water.Viscosity.Type = 'Constant'
+overland.Phase.water.Viscosity.Type = "Constant"
 overland.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Contaminants.Names = ''
+overland.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Geom.Retardation.GeomNames = ''
+overland.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 overland.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 overland.TimingInfo.BaseUnit = 0.05
 overland.TimingInfo.StartCount = 0
 overland.TimingInfo.StartTime = 0.0
 overland.TimingInfo.StopTime = 1.0
 overland.TimingInfo.DumpInterval = -2
-overland.TimeStep.Type = 'Constant'
+overland.TimeStep.Type = "Constant"
 overland.TimeStep.Value = 0.05
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Geom.Porosity.GeomNames = 'domain'
-overland.Geom.domain.Porosity.Type = 'Constant'
+overland.Geom.Porosity.GeomNames = "domain"
+overland.Geom.domain.Porosity.Type = "Constant"
 overland.Geom.domain.Porosity.Value = 0.01
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Domain.GeomName = 'domain'
+overland.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Phase.RelPerm.Type = 'VanGenuchten'
-overland.Phase.RelPerm.GeomNames = 'domain'
+overland.Phase.RelPerm.Type = "VanGenuchten"
+overland.Phase.RelPerm.GeomNames = "domain"
 
 overland.Geom.domain.RelPerm.Alpha = 6.0
-overland.Geom.domain.RelPerm.N = 2.
+overland.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland.Phase.Saturation.Type = 'VanGenuchten'
-overland.Phase.Saturation.GeomNames = 'domain'
+overland.Phase.Saturation.Type = "VanGenuchten"
+overland.Phase.Saturation.GeomNames = "domain"
 
 overland.Geom.domain.Saturation.Alpha = 6.0
-overland.Geom.domain.Saturation.N = 2.
+overland.Geom.domain.Saturation.N = 2.0
 overland.Geom.domain.Saturation.SRes = 0.2
 overland.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Wells.Names = ''
+overland.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Cycle.Names = 'constant rainrec'
-overland.Cycle.constant.Names = 'alltime'
+overland.Cycle.Names = "constant rainrec"
+overland.Cycle.constant.Names = "alltime"
 overland.Cycle.constant.alltime.Length = 1
 overland.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-overland.Cycle.rainrec.Names = 'rain rec'
+overland.Cycle.rainrec.Names = "rain rec"
 overland.Cycle.rainrec.rain.Length = 2
 overland.Cycle.rainrec.rec.Length = 300
 overland.Cycle.rainrec.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 overland.BCPressure.PatchNames = overland.Geom.domain.Patches
 
-overland.Patch.x_lower.BCPressure.Type = 'FluxConst'
-overland.Patch.x_lower.BCPressure.Cycle = 'constant'
+overland.Patch.x_lower.BCPressure.Type = "FluxConst"
+overland.Patch.x_lower.BCPressure.Cycle = "constant"
 overland.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-overland.Patch.y_lower.BCPressure.Type = 'FluxConst'
-overland.Patch.y_lower.BCPressure.Cycle = 'constant'
+overland.Patch.y_lower.BCPressure.Type = "FluxConst"
+overland.Patch.y_lower.BCPressure.Cycle = "constant"
 overland.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-overland.Patch.z_lower.BCPressure.Type = 'FluxConst'
-overland.Patch.z_lower.BCPressure.Cycle = 'constant'
+overland.Patch.z_lower.BCPressure.Type = "FluxConst"
+overland.Patch.z_lower.BCPressure.Cycle = "constant"
 overland.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-overland.Patch.x_upper.BCPressure.Type = 'FluxConst'
-overland.Patch.x_upper.BCPressure.Cycle = 'constant'
+overland.Patch.x_upper.BCPressure.Type = "FluxConst"
+overland.Patch.x_upper.BCPressure.Cycle = "constant"
 overland.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-overland.Patch.y_upper.BCPressure.Type = 'FluxConst'
-overland.Patch.y_upper.BCPressure.Cycle = 'constant'
+overland.Patch.y_upper.BCPressure.Type = "FluxConst"
+overland.Patch.y_upper.BCPressure.Cycle = "constant"
 overland.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
 ## overland flow boundary condition with very heavy rainfall then slight ET
-overland.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-overland.Patch.z_upper.BCPressure.Cycle = 'rainrec'
+overland.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+overland.Patch.z_upper.BCPressure.Cycle = "rainrec"
 overland.Patch.z_upper.BCPressure.rain.Value = -0.01
 overland.Patch.z_upper.BCPressure.rec.Value = 0.0000
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland.Mannings.Type = 'Constant'
-overland.Mannings.GeomNames = 'domain'
-overland.Mannings.Geom.domain.Value = 3.e-6
+overland.Mannings.Type = "Constant"
+overland.Mannings.GeomNames = "domain"
+overland.Mannings.Geom.domain.Value = 3.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.PhaseSources.water.Type = 'Constant'
-overland.PhaseSources.water.GeomNames = 'domain'
+overland.PhaseSources.water.Type = "Constant"
+overland.PhaseSources.water.GeomNames = "domain"
 overland.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.KnownSolution = 'NoKnownSolution'
+overland.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Solver = 'Richards'
+overland.Solver = "Richards"
 overland.Solver.MaxIter = 2500
 
 overland.Solver.Nonlinear.MaxIter = 50
 overland.Solver.Nonlinear.ResidualTol = 1e-9
-overland.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+overland.Solver.Nonlinear.EtaChoice = "EtaConstant"
 overland.Solver.Nonlinear.EtaValue = 0.01
 overland.Solver.Nonlinear.UseJacobian = False
 
 overland.Solver.Nonlinear.DerivativeEpsilon = 1e-15
 overland.Solver.Nonlinear.StepTol = 1e-20
-overland.Solver.Nonlinear.Globalization = 'LineSearch'
+overland.Solver.Nonlinear.Globalization = "LineSearch"
 overland.Solver.Linear.KrylovDimension = 20
 overland.Solver.Linear.MaxRestart = 2
 
-overland.Solver.Linear.Preconditioner = 'PFMG'
+overland.Solver.Linear.Preconditioner = "PFMG"
 overland.Solver.PrintSubsurf = False
-overland.Solver.Drop = 1E-20
-overland.Solver.AbsTol = 1E-10
+overland.Solver.Drop = 1e-20
+overland.Solver.AbsTol = 1e-10
 
-overland.Solver.OverlandKinematic.Epsilon = 1E-5
+overland.Solver.OverlandKinematic.Epsilon = 1e-5
 
 overland.Solver.WriteSiloSubsurfData = False
 overland.Solver.WriteSiloPressure = False
@@ -266,212 +274,255 @@
 overland.Solver.WriteSiloSaturation = False
 overland.Solver.WriteSiloConcentration = False
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be at the bottom of the domain, the top layer is initially dry
-overland.ICPressure.Type = 'HydroStaticPatch'
-overland.ICPressure.GeomNames = 'domain'
+overland.ICPressure.Type = "HydroStaticPatch"
+overland.ICPressure.GeomNames = "domain"
 overland.Geom.domain.ICPressure.Value = -3.0
 
-overland.Geom.domain.ICPressure.RefGeom = 'domain'
-overland.Geom.domain.ICPressure.RefPatch = 'z_upper'
+overland.Geom.domain.ICPressure.RefGeom = "domain"
+overland.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
 # Running all 8 direction combinations with the upwind formulation on and off (i.e. 16 total)
 # Commented lines are from original TCL test - will need to convert to Python if running with
 # Python pftools
-#-----------------------------------------------------------------------------
-#set runcheck to 1 if you want to run the pass fail tests
-# runcheck = 1
-# first = 1
-# source pftest.tcl
+# -----------------------------------------------------------------------------
+# set runcheck to 1 if you want to run the pass fail tests
+runcheck = 1
+first = 1
+correct_output_dir_name = get_absolute_path("../correct_output")
 
 ###############################
 # Looping over slop configurations
 ###############################
-# foreach xslope [list 0.01 -0.01] yslope [list 0.01 -0.01] name [list posxposy negxnegy] {
-#   puts "$xslope $yslope $name"
-#set name negy
-
-#   #### Set the slopes
-overland.TopoSlopesX.Type = "Constant"
-overland.TopoSlopesX.GeomNames = "domain"
-overland.TopoSlopesX.Geom.domain.Value = 0.01
-
-overland.TopoSlopesY.Type = "Constant"
-overland.TopoSlopesY.GeomNames = "domain"
-overland.TopoSlopesY.Geom.domain.Value = 0.01
-
-#   #original approach from K&M AWR 2006
-overland.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-overland.Solver.Nonlinear.UseJacobian = False
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'PFSymmetric'
-
-
-#   set runname Slab.$name.OverlandModule
-#   puts "##########"
-#   if $first==1 {
-#     puts $runname
-#     set first 0
-#   } else {
-#     puts "Running $runname Jacobian True"
-#   }
-#   pfrun $runname
-#   pfundist $runname
-#   if $runcheck==1 {
-#     set passed 1
-#     foreach i "00000 00001 00002 00003 00004 00005 00006 00007 00008 00009 00010" {
-#       if ![pftestFile $runname.out.press.$i.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
-#         set passed 0
-#       }
-#       if ![pftestFile  $runname.out.satur.$i.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
-#         set passed 0
-#       }
-#     }
-#     if $passed {
-#       puts "$runname : PASSED"
-#     } {
-#       puts "$runname : FAILED"
-#     }
-#   }
-
-# turn on analytical jacobian and re-test
-overland.Solver.Nonlinear.UseJacobian = True
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'PFSymmetric'
-
-# set runname Slab.$name.OverlandModule
-# puts "##########"
-# puts "Running $runname Jacobian True"
-# pfrun $runname
-# pfundist $runname
-# if $runcheck==1 {
-#   set passed 1
-#   foreach i "00000 00001 00002 00003 00004 00005 00006 00007 00008 00009 00010" {
-#     if ![pftestFile $runname.out.press.$i.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#     if ![pftestFile  $runname.out.satur.$i.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#   }
-#   if $passed {
-#     puts "$runname : PASSED"
-#   } {
-#     puts "$runname : FAILED"
-#   }
-# }
-
-# turn on non-symmetric Preconditioner and re-test
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'FullJacobian'
-
-# set runname Slab.$name.OverlandModule
-# puts "##########"
-# puts "Running $runname Jacobian True Nonsymmetric Preconditioner"
-# pfrun $runname
-# pfundist $runname
-# if $runcheck==1 {
-#   set passed 1
-#   foreach i "00000 00001 00002 00003 00004 00005 00006 00007 00008 00009 00010" {
-#     if ![pftestFile $runname.out.press.$i.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#     if ![pftestFile  $runname.out.satur.$i.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#   }
-#   if $passed {
-#     puts "$runname : PASSED"
-#   } {
-#     puts "$runname : FAILED"
-#   }
-# }
-
-# run with KWE upwinding
-overland.Patch.z_upper.BCPressure.Type = 'OverlandKinematic'
-overland.Solver.Nonlinear.UseJacobian = False
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'PFSymmetric'
-
-#   set runname Slab.$name.OverlandKin
-#   puts "##########"
-#   puts "Running $runname"
-#   pfrun $runname
-#   pfundist $runname
-#   if $runcheck==1 {
-#     set passed 1
-#     foreach i "00000 00001 00002 00003 00004 00005 00006 00007 00008 00009 00010" {
-#       if ![pftestFile $runname.out.press.$i.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
-#         set passed 0
-#       }
-#       if ![pftestFile  $runname.out.satur.$i.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
-#         set passed 0
-#       }
-#     }
-#     if $passed {
-#       puts "$runname : PASSED"
-#     } {
-#       puts "$runname  : FAILED"
-#     }
-#   }
-
-#   # run with KWE upwinding jacobian true
-#   pfset Patch.z-upper.BCPressure.Type		      OverlandKinematic
-#   pfset Solver.Nonlinear.UseJacobian                       True
-#   pfset Solver.Linear.Preconditioner.PCMatrixType PFSymmetric
-
-#     set runname Slab.$name.OverlandKin
-#     puts "##########"
-#     puts "Running $runname Jacobian True"
-#     pfrun $runname
-#     pfundist $runname
-#     if $runcheck==1 {
-#       set passed 1
-#       foreach i "00000 00001 00002 00003 00004 00005 00006 00007 00008 00009 00010" {
-#         if ![pftestFile $runname.out.press.$i.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
-#           set passed 0
-#         }
-#         if ![pftestFile  $runname.out.satur.$i.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
-#           set passed 0
-#         }
-#       }
-#       if $passed {
-#         puts "$runname : PASSED"
-#       } {
-#         puts "$runname  : FAILED"
-#       }
-#     }
-
-#     # run with KWE upwinding jacobian true and nonsymmetric preconditioner
-#     pfset Patch.z-upper.BCPressure.Type		      OverlandKinematic
-#     pfset Solver.Nonlinear.UseJacobian                       True
-#     pfset Solver.Linear.Preconditioner.PCMatrixType FullJacobian
-
-#       set runname Slab.$name.OverlandKin
-#       puts "##########"
-#       puts "Running $runname Jacobian True Nonsymmetric Preconditioner"
-#       pfrun $runname
-#       pfundist $runname
-#       if $runcheck==1 {
-#         set passed 1
-#         foreach i "00000 00001 00002 00003 00004 00005 00006 00007 00008 00009 00010" {
-#           if ![pftestFile $runname.out.press.$i.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
-#             set passed 0
-#           }
-#           if ![pftestFile  $runname.out.satur.$i.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
-#             set passed 0
-#           }
-#         }
-#         if $passed {
-#           puts "$runname : PASSED"
-#         } {
-#           puts "$runname  : FAILED"
-#         }
-#       }
-# }
-
-dir_name = get_absolute_path('test_output/os_kwe')
-mkdir(dir_name)
-overland.run(working_directory=dir_name)
-
+x_slopes = [0.01, -0.01]
+y_slopes = [0.01, -0.01]
+names = ["posxposy", "negxnegy"]
+
+for x_slope, y_slope, name in zip(x_slopes, y_slopes, names):
+    print(f"{x_slope} {y_slope} {name}")
+    overland.TopoSlopesX.Type = "Constant"
+    overland.TopoSlopesX.GeomNames = "domain"
+    overland.TopoSlopesX.Geom.domain.Value = x_slope
+
+    overland.TopoSlopesY.Type = "Constant"
+    overland.TopoSlopesY.GeomNames = "domain"
+    overland.TopoSlopesY.Geom.domain.Value = y_slope
+
+    overland.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+    overland.Solver.Nonlinear.UseJacobian = False
+    overland.Solver.Linear.Preconditioner.PCMatrixType = "PFSymmetric"
+
+    run_name = f"Slab.{name}.OverlandModule"
+    overland.set_name(run_name)
+    print("##########")
+    if first == 1:
+        print(overland.get_name())
+        first = 0
+    else:
+        print(f"Running {run_name} Jacobian False")
+    new_output_dir_name = get_absolute_path("test_output/" + f"run_name_jacobian_false")
+    mkdir(new_output_dir_name)
+    overland.run(working_directory=new_output_dir_name)
+    if runcheck == 1:
+        passed = True
+        for i in range(11):
+            timestep = str(i).rjust(5, "0")
+            filename = f"/{run_name}.out.press.{timestep}.pfb"
+            if not pf_test_file(
+                new_output_dir_name + filename,
+                correct_output_dir_name + filename,
+                f"Max difference in Pressure for timestep {timestep}",
+            ):
+                passed = False
+            filename = f"/{run_name}.out.satur.{timestep}.pfb"
+            if not pf_test_file(
+                new_output_dir_name + filename,
+                correct_output_dir_name + filename,
+                f"Max difference in Saturation for timestep {timestep}",
+            ):
+                passed = False
+
+        if passed:
+            print(f"{run_name} : PASSED")
+        else:
+            print(f"{run_name} : FAILED")
+            sys.exit(1)
+    rm(new_output_dir_name)
+
+    overland.Solver.Nonlinear.UseJacobian = True
+    overland.Solver.Linear.Preconditioner.PCMatrixType = "PFSymmetric"
+    print("##########")
+    print(f"Running {run_name} Jacobian True")
+    new_output_dir_name = get_absolute_path("test_output/" + f"run_name_jacobian_true")
+    mkdir(new_output_dir_name)
+    overland.run(working_directory=new_output_dir_name)
+    if runcheck == 1:
+        passed = True
+        for i in range(11):
+            timestep = str(i).rjust(5, "0")
+            filename = f"/{run_name}.out.press.{timestep}.pfb"
+            if not pf_test_file(
+                new_output_dir_name + filename,
+                correct_output_dir_name + filename,
+                f"Max difference in Pressure for timestep {timestep}",
+            ):
+                passed = False
+            filename = f"/{run_name}.out.satur.{timestep}.pfb"
+            if not pf_test_file(
+                new_output_dir_name + filename,
+                correct_output_dir_name + filename,
+                f"Max difference in Saturation for timestep {timestep}",
+            ):
+                passed = False
+
+        if passed:
+            print(f"{run_name} : PASSED")
+        else:
+            print(f"{run_name} : FAILED")
+            sys.exit(1)
+    rm(new_output_dir_name)
+
+    overland.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
+    print("##########")
+    print(f"Running {run_name} Jacobian True Nonsymmetric Preconditioner")
+    new_output_dir_name = get_absolute_path(
+        "test_output/" + f"run_name_nonsymmetric_preconditioner"
+    )
+    mkdir(new_output_dir_name)
+    overland.run(working_directory=new_output_dir_name)
+    if runcheck == 1:
+        passed = True
+        for i in range(11):
+            timestep = str(i).rjust(5, "0")
+            filename = f"/{run_name}.out.press.{timestep}.pfb"
+            if not pf_test_file(
+                new_output_dir_name + filename,
+                correct_output_dir_name + filename,
+                f"Max difference in Pressure for timestep {timestep}",
+            ):
+                passed = False
+            filename = f"/{run_name}.out.satur.{timestep}.pfb"
+            if not pf_test_file(
+                new_output_dir_name + filename,
+                correct_output_dir_name + filename,
+                f"Max difference in Saturation for timestep {timestep}",
+            ):
+                passed = False
+        if passed:
+            print(f"{run_name} : PASSED")
+        else:
+            print(f"{run_name} : FAILED")
+            sys.exit(1)
+    rm(new_output_dir_name)
+
+    overland.Patch.z_upper.BCPressure.Type = "OverlandKinematic"
+    overland.Solver.Nonlinear.UseJacobian = False
+    overland.Solver.Linear.Preconditioner.PCMatrixType = "PFSymmetric"
+    run_name = f"Slab.{name}.OverlandKin"
+    overland.set_name(run_name)
+    print("##########")
+    print(f"Running {run_name}")
+    new_output_dir_name = get_absolute_path("test_output/" + f"run_name_overland_kin")
+    mkdir(new_output_dir_name)
+    overland.run(working_directory=new_output_dir_name)
+    if runcheck == 1:
+        passed = True
+        for i in range(11):
+            timestep = str(i).rjust(5, "0")
+            filename = f"/{run_name}.out.press.{timestep}.pfb"
+            if not pf_test_file(
+                new_output_dir_name + filename,
+                correct_output_dir_name + filename,
+                f"Max difference in Pressure for timestep {timestep}",
+            ):
+                passed = False
+            filename = f"/{run_name}.out.satur.{timestep}.pfb"
+            if not pf_test_file(
+                new_output_dir_name + filename,
+                correct_output_dir_name + filename,
+                f"Max difference in Saturation for timestep {timestep}",
+            ):
+                passed = False
+        if passed:
+            print(f"{run_name} : PASSED")
+        else:
+            print(f"{run_name} : FAILED")
+            sys.exit(1)
+    rm(new_output_dir_name)
+
+    overland.Patch.z_upper.BCPressure.Type = "OverlandKinematic"
+    overland.Solver.Nonlinear.UseJacobian = True
+    overland.Solver.Linear.Preconditioner.PCMatrixType = "PFSymmetric"
+    print("##########")
+    print(f"Running {run_name} Jacobian True")
+    new_output_dir_name = get_absolute_path(
+        "test_output/" + f"run_name_overland_kin_jacobian_true"
+    )
+    mkdir(new_output_dir_name)
+    overland.run(working_directory=new_output_dir_name)
+    if runcheck == 1:
+        passed = True
+        for i in range(11):
+            timestep = str(i).rjust(5, "0")
+            filename = f"/{run_name}.out.press.{timestep}.pfb"
+            if not pf_test_file(
+                new_output_dir_name + filename,
+                correct_output_dir_name + filename,
+                f"Max difference in Pressure for timestep {timestep}",
+            ):
+                passed = False
+            filename = f"/{run_name}.out.satur.{timestep}.pfb"
+            if not pf_test_file(
+                new_output_dir_name + filename,
+                correct_output_dir_name + filename,
+                f"Max difference in Saturation for timestep {timestep}",
+            ):
+                passed = False
+        if passed:
+            print(f"{run_name} : PASSED")
+        else:
+            print(f"{run_name} : FAILED")
+            sys.exit(1)
+    rm(new_output_dir_name)
+
+    overland.Patch.z_upper.BCPressure.Type = "OverlandKinematic"
+    overland.Solver.Nonlinear.UseJacobian = True
+    overland.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
+    print("##########")
+    print(f"Running {run_name} Jacobian True Nonsymmetric Preconditioner")
+    new_output_dir_name = get_absolute_path(
+        "test_output/"
+        + f"run_name_overland_kin_jacobian_true_nonsymmetric_preconditioner"
+    )
+    mkdir(new_output_dir_name)
+    overland.run(working_directory=new_output_dir_name)
+    if runcheck == 1:
+        passed = True
+        for i in range(11):
+            timestep = str(i).rjust(5, "0")
+            filename = f"/{run_name}.out.press.{timestep}.pfb"
+            if not pf_test_file(
+                new_output_dir_name + filename,
+                correct_output_dir_name + filename,
+                f"Max difference in Pressure for timestep {timestep}",
+            ):
+                passed = False
+            filename = f"/{run_name}.out.satur.{timestep}.pfb"
+            if not pf_test_file(
+                new_output_dir_name + filename,
+                correct_output_dir_name + filename,
+                f"Max difference in Saturation for timestep {timestep}",
+            ):
+                passed = False
+        if passed:
+            print(f"{run_name} : PASSED")
+        else:
+            print(f"{run_name} : FAILED")
+            sys.exit(1)
+    rm(new_output_dir_name)
diff --git a/test/python/overland_tiltedV_DWE.py b/test/python/overland_tiltedV_DWE.py
index 58ea3346b..02301e5bf 100644
--- a/test/python/overland_tiltedV_DWE.py
+++ b/test/python/overland_tiltedV_DWE.py
@@ -1,13 +1,15 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # running different configuraitons of tilted V
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
 overland = Run("overland_tiltedV_DWE", __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.FileVersion = 4
 
@@ -15,9 +17,9 @@
 overland.Process.Topology.Q = 1
 overland.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.ComputationalGrid.Lower.X = 0.0
 overland.ComputationalGrid.Lower.Y = 0.0
@@ -29,27 +31,27 @@
 
 overland.ComputationalGrid.DX = 10.0
 overland.ComputationalGrid.DY = 10.0
-overland.ComputationalGrid.DZ = .05
+overland.ComputationalGrid.DZ = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland.GeomInput.Names = 'domaininput leftinput rightinput channelinput'
+overland.GeomInput.Names = "domaininput leftinput rightinput channelinput"
 
-overland.GeomInput.domaininput.GeomName = 'domain'
-overland.GeomInput.leftinput.GeomName = 'left'
-overland.GeomInput.rightinput.GeomName = 'right'
-overland.GeomInput.channelinput.GeomName = 'channel'
+overland.GeomInput.domaininput.GeomName = "domain"
+overland.GeomInput.leftinput.GeomName = "left"
+overland.GeomInput.rightinput.GeomName = "right"
+overland.GeomInput.channelinput.GeomName = "channel"
 
-overland.GeomInput.domaininput.InputType = 'Box'
-overland.GeomInput.leftinput.InputType = 'Box'
-overland.GeomInput.rightinput.InputType = 'Box'
-overland.GeomInput.channelinput.InputType = 'Box'
+overland.GeomInput.domaininput.InputType = "Box"
+overland.GeomInput.leftinput.InputType = "Box"
+overland.GeomInput.rightinput.InputType = "Box"
+overland.GeomInput.channelinput.InputType = "Box"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.Geom.domain.Lower.X = 0.0
 overland.Geom.domain.Lower.Y = 0.0
@@ -58,11 +60,11 @@
 overland.Geom.domain.Upper.X = 50.0
 overland.Geom.domain.Upper.Y = 50.0
 overland.Geom.domain.Upper.Z = 0.05
-overland.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+overland.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Left Slope Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.Geom.left.Lower.X = 0.0
 overland.Geom.left.Lower.Y = 0.0
@@ -72,9 +74,9 @@
 overland.Geom.left.Upper.Y = 50.0
 overland.Geom.left.Upper.Z = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Right Slope Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.Geom.right.Lower.X = 30.0
 overland.Geom.right.Lower.Y = 0.0
@@ -84,9 +86,9 @@
 overland.Geom.right.Upper.Y = 50.0
 overland.Geom.right.Upper.Z = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Channel Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.Geom.channel.Lower.X = 20.0
 overland.Geom.channel.Lower.Y = 0.0
@@ -96,206 +98,206 @@
 overland.Geom.channel.Upper.Y = 50.0
 overland.Geom.channel.Upper.Z = 0.05
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Geom.Perm.Names = 'domain'
-overland.Geom.domain.Perm.Type = 'Constant'
+overland.Geom.Perm.Names = "domain"
+overland.Geom.domain.Perm.Type = "Constant"
 overland.Geom.domain.Perm.Value = 0.0000001
 
-overland.Perm.TensorType = 'TensorByGeom'
+overland.Perm.TensorType = "TensorByGeom"
 
-overland.Geom.Perm.TensorByGeom.Names = 'domain'
+overland.Geom.Perm.TensorByGeom.Names = "domain"
 
 overland.Geom.domain.Perm.TensorValX = 1.0
 overland.Geom.domain.Perm.TensorValY = 1.0
 overland.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.SpecificStorage.Type = 'Constant'
-overland.SpecificStorage.GeomNames = 'domain'
+overland.SpecificStorage.Type = "Constant"
+overland.SpecificStorage.GeomNames = "domain"
 overland.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Phase.Names = 'water'
+overland.Phase.Names = "water"
 
-overland.Phase.water.Density.Type = 'Constant'
+overland.Phase.water.Density.Type = "Constant"
 overland.Phase.water.Density.Value = 1.0
 
-overland.Phase.water.Viscosity.Type = 'Constant'
+overland.Phase.water.Viscosity.Type = "Constant"
 overland.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Contaminants.Names = ''
+overland.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Geom.Retardation.GeomNames = ''
+overland.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 overland.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 overland.TimingInfo.BaseUnit = 0.05
 overland.TimingInfo.StartCount = 0
 overland.TimingInfo.StartTime = 0.0
 overland.TimingInfo.StopTime = 2.0
 overland.TimingInfo.DumpInterval = -2
-overland.TimeStep.Type = 'Constant'
+overland.TimeStep.Type = "Constant"
 overland.TimeStep.Value = 0.05
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Geom.Porosity.GeomNames = 'domain'
-overland.Geom.domain.Porosity.Type = 'Constant'
+overland.Geom.Porosity.GeomNames = "domain"
+overland.Geom.domain.Porosity.Type = "Constant"
 overland.Geom.domain.Porosity.Value = 0.01
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Domain.GeomName = 'domain'
+overland.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Phase.RelPerm.Type = 'VanGenuchten'
-overland.Phase.RelPerm.GeomNames = 'domain'
+overland.Phase.RelPerm.Type = "VanGenuchten"
+overland.Phase.RelPerm.GeomNames = "domain"
 
 overland.Geom.domain.RelPerm.Alpha = 6.0
-overland.Geom.domain.RelPerm.N = 2.
+overland.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland.Phase.Saturation.Type = 'VanGenuchten'
-overland.Phase.Saturation.GeomNames = 'domain'
+overland.Phase.Saturation.Type = "VanGenuchten"
+overland.Phase.Saturation.GeomNames = "domain"
 
 overland.Geom.domain.Saturation.Alpha = 6.0
-overland.Geom.domain.Saturation.N = 2.
+overland.Geom.domain.Saturation.N = 2.0
 overland.Geom.domain.Saturation.SRes = 0.2
 overland.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
-overland.Wells.Names = ''
+# -----------------------------------------------------------------------------
+overland.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Cycle.Names = 'constant rainrec'
-overland.Cycle.constant.Names = 'alltime'
+overland.Cycle.Names = "constant rainrec"
+overland.Cycle.constant.Names = "alltime"
 overland.Cycle.constant.alltime.Length = 1
 overland.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-overland.Cycle.rainrec.Names = 'rain rec'
+overland.Cycle.rainrec.Names = "rain rec"
 overland.Cycle.rainrec.rain.Length = 2
 overland.Cycle.rainrec.rec.Length = 300
 overland.Cycle.rainrec.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 overland.BCPressure.PatchNames = overland.Geom.domain.Patches
 
-overland.Patch.x_lower.BCPressure.Type = 'FluxConst'
-overland.Patch.x_lower.BCPressure.Cycle = 'constant'
+overland.Patch.x_lower.BCPressure.Type = "FluxConst"
+overland.Patch.x_lower.BCPressure.Cycle = "constant"
 overland.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-overland.Patch.y_lower.BCPressure.Type = 'FluxConst'
-overland.Patch.y_lower.BCPressure.Cycle = 'constant'
+overland.Patch.y_lower.BCPressure.Type = "FluxConst"
+overland.Patch.y_lower.BCPressure.Cycle = "constant"
 overland.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-overland.Patch.z_lower.BCPressure.Type = 'FluxConst'
-overland.Patch.z_lower.BCPressure.Cycle = 'constant'
+overland.Patch.z_lower.BCPressure.Type = "FluxConst"
+overland.Patch.z_lower.BCPressure.Cycle = "constant"
 overland.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-overland.Patch.x_upper.BCPressure.Type = 'FluxConst'
-overland.Patch.x_upper.BCPressure.Cycle = 'constant'
+overland.Patch.x_upper.BCPressure.Type = "FluxConst"
+overland.Patch.x_upper.BCPressure.Cycle = "constant"
 overland.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-overland.Patch.y_upper.BCPressure.Type = 'FluxConst'
-overland.Patch.y_upper.BCPressure.Cycle = 'constant'
+overland.Patch.y_upper.BCPressure.Type = "FluxConst"
+overland.Patch.y_upper.BCPressure.Cycle = "constant"
 overland.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
 ## overland flow boundary condition with very heavy rainfall
-overland.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-overland.Patch.z_upper.BCPressure.Cycle = 'rainrec'
+overland.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+overland.Patch.z_upper.BCPressure.Cycle = "rainrec"
 overland.Patch.z_upper.BCPressure.rain.Value = -0.01
 overland.Patch.z_upper.BCPressure.rec.Value = 0.0000
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland.Mannings.Type = 'Constant'
-overland.Mannings.GeomNames = 'domain'
-overland.Mannings.Geom.domain.Value = 3.e-6
+overland.Mannings.Type = "Constant"
+overland.Mannings.GeomNames = "domain"
+overland.Mannings.Geom.domain.Value = 3.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.PhaseSources.water.Type = 'Constant'
-overland.PhaseSources.water.GeomNames = 'domain'
+overland.PhaseSources.water.Type = "Constant"
+overland.PhaseSources.water.GeomNames = "domain"
 overland.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.KnownSolution = 'NoKnownSolution'
+overland.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Solver = 'Richards'
+overland.Solver = "Richards"
 overland.Solver.MaxIter = 2500
 
 overland.Solver.Nonlinear.MaxIter = 100
 overland.Solver.Nonlinear.ResidualTol = 1e-9
-overland.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+overland.Solver.Nonlinear.EtaChoice = "EtaConstant"
 overland.Solver.Nonlinear.EtaValue = 0.01
 overland.Solver.Nonlinear.UseJacobian = False
 overland.Solver.Nonlinear.DerivativeEpsilon = 1e-15
 overland.Solver.Nonlinear.StepTol = 1e-20
-overland.Solver.Nonlinear.Globalization = 'LineSearch'
+overland.Solver.Nonlinear.Globalization = "LineSearch"
 overland.Solver.Linear.KrylovDimension = 50
 overland.Solver.Linear.MaxRestart = 2
 
-overland.Solver.Linear.Preconditioner = 'PFMG'
+overland.Solver.Linear.Preconditioner = "PFMG"
 overland.Solver.PrintSubsurf = False
-overland.Solver.Drop = 1E-20
-overland.Solver.AbsTol = 1E-10
+overland.Solver.Drop = 1e-20
+overland.Solver.AbsTol = 1e-10
 
 overland.Solver.WriteSiloSubsurfData = False
 overland.Solver.WriteSiloPressure = False
@@ -304,28 +306,28 @@
 overland.Solver.WriteSiloSaturation = False
 overland.Solver.WriteSiloConcentration = False
 
-overland.Solver.OverlandDiffusive.Epsilon = 1E-5
+overland.Solver.OverlandDiffusive.Epsilon = 1e-5
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be at the bottom of the domain, the top layer is initially dry
-overland.ICPressure.Type = 'HydroStaticPatch'
-overland.ICPressure.GeomNames = 'domain'
+overland.ICPressure.Type = "HydroStaticPatch"
+overland.ICPressure.GeomNames = "domain"
 overland.Geom.domain.ICPressure.Value = -3.0
 
-overland.Geom.domain.ICPressure.RefGeom = 'domain'
-overland.Geom.domain.ICPressure.RefPatch = 'z_upper'
+overland.Geom.domain.ICPressure.RefGeom = "domain"
+overland.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-#set runcheck to 1 if you want to run the pass fail tests
-# runcheck = 1
-# source pftest.tcl
+# -----------------------------------------------------------------------------
+# set runcheck to 1 if you want to run the pass fail tests
+runcheck = 1
 
-#-----------------------------------------------------------------------------
+
+# -----------------------------------------------------------------------------
 # New diffusive formulations without the zero channel (as compared to the first
 #    tests in overland_tiltedV_KWE.tcl)
 # Note: The difference in configuration here is to be consistent with the way
@@ -333,99 +335,125 @@
 #   These two results should be almost identical for the new and old formulations
 # Commented lines are from original TCL test - will need to convert to Python if running with
 # Python pftools
-#-----------------------------------------------------------------------------
-overland.TopoSlopesX.Type = 'Constant'
-overland.TopoSlopesX.GeomNames = 'left right channel'
+# -----------------------------------------------------------------------------
+overland.TopoSlopesX.Type = "Constant"
+overland.TopoSlopesX.GeomNames = "left right channel"
 overland.TopoSlopesX.Geom.left.Value = -0.01
 overland.TopoSlopesX.Geom.right.Value = 0.01
 overland.TopoSlopesX.Geom.channel.Value = 0.01
 
-overland.TopoSlopesY.Type = 'Constant'
-overland.TopoSlopesY.GeomNames = 'domain'
+overland.TopoSlopesY.Type = "Constant"
+overland.TopoSlopesY.GeomNames = "domain"
 overland.TopoSlopesY.Geom.domain.Value = 0.01
 
 # run with DWE
-overland.Patch.z_upper.BCPressure.Type = 'OverlandDiffusive'
+overland.Patch.z_upper.BCPressure.Type = "OverlandDiffusive"
 overland.Solver.Nonlinear.UseJacobian = False
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'PFSymmetric'
-
-# set runname TiltedV_OverlandDif
-# puts "##########"
-# puts $runname
-# pfrun $runname
-# pfundist $runname
-# if $runcheck==1 {
-#   set passed 1
-#   foreach i "00000 00001 00002 00003 00004 00005 00006 00007 00008 00009 00010" {
-#     if ![pftestFile $runname.out.press.$i.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#     if ![pftestFile  $runname.out.satur.$i.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#   }
-#   if $passed {
-#     puts "$runname : PASSED"
-#   } {
-#     puts "$runname : FAILED"
-#   }
-# }
+overland.Solver.Linear.Preconditioner.PCMatrixType = "PFSymmetric"
+
+run_name = "TiltedV_OverlandDif"
+overland.set_name(run_name)
+correct_output_dir_name = get_absolute_path("../correct_output")
+
+print("##########")
+print(run_name)
+new_output_dir_name = get_absolute_path("test_output/" + run_name)
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+
+if runcheck == 1:
+    passed = True
+    for i in range(11):
+        timestep = str(i).rjust(5, "0")
+        filename = f"/{run_name}.out.press.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Pressure for timestep {timestep}",
+        ):
+            passed = False
+        filename = f"/{run_name}.out.satur.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Saturation for timestep {timestep}",
+        ):
+            passed = False
+
+    if passed:
+        print(f"{run_name} : PASSED")
+    else:
+        print(f"{run_name} : FAILED")
+        sys.exit(1)
+
 
 # run with KWE upwinding and analytical jacobian
-overland.Patch.z_upper.BCPressure.Type = 'OverlandDiffusive'
+overland.Patch.z_upper.BCPressure.Type = "OverlandDiffusive"
 overland.Solver.Nonlinear.UseJacobian = True
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'PFSymmetric'
-
-# set runname TiltedV_OverlandDif
-# puts "##########"
-# puts "Running $runname Jacobian True"
-# pfrun $runname
-# pfundist $runname
-# if $runcheck==1 {
-#   set passed 1
-#   foreach i "00000 00001 00002 00003 00004 00005 00006 00007 00008 00009 00010" {
-#     if ![pftestFile $runname.out.press.$i.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#     if ![pftestFile  $runname.out.satur.$i.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#   }
-#   if $passed {
-#     puts "$runname : PASSED"
-#   } {
-#     puts "$runname : FAILED"
-#   }
-# }
+overland.Solver.Linear.Preconditioner.PCMatrixType = "PFSymmetric"
+
+print("##########")
+print(f"Running {run_name} Jacobian True")
+new_output_dir_name = get_absolute_path("test_output/" + f"{run_name}_jacobian_true")
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if runcheck == 1:
+    passed = True
+    for i in range(11):
+        timestep = str(i).rjust(5, "0")
+        filename = f"/{run_name}.out.press.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Pressure for timestep {timestep}",
+        ):
+            passed = False
+        filename = f"/{run_name}.out.satur.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Saturation for timestep {timestep}",
+        ):
+            passed = False
+    if passed:
+        print(f"{run_name} : PASSED")
+    else:
+        print(f"{run_name} : FAILED")
+        sys.exit(1)
+
 
 # run with KWE upwinding and analytical jacobian and nonsymmetric preconditioner
-overland.Patch.z_upper.BCPressure.Type = 'OverlandDiffusive'
+overland.Patch.z_upper.BCPressure.Type = "OverlandDiffusive"
 overland.Solver.Nonlinear.UseJacobian = True
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'FullJacobian'
-
-# set runname TiltedV_OverlandDif
-# puts "##########"
-# puts "Running $runname Jacobian True Nonsymmetric Preconditioner"
-# pfrun $runname
-# pfundist $runname
-# if $runcheck==1 {
-#   set passed 1
-#   foreach i "00000 00001 00002 00003 00004 00005 00006 00007 00008 00009 00010" {
-#     if ![pftestFile $runname.out.press.$i.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#     if ![pftestFile  $runname.out.satur.$i.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#   }
-#   if $passed {
-#     puts "$runname : PASSED"
-#   } {
-#     puts "$runname : FAILED"
-#   }
-# }
-
-dir_name = get_absolute_path('test_output/ov_dwe')
-mkdir(dir_name)
-overland.run(working_directory=dir_name)
-
+overland.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
+
+print("##########")
+print(f"Running {run_name} Jacobian True Nonsymmetric Preconditioner")
+new_output_dir_name = get_absolute_path(
+    "test_output/" + f"{run_name}_jacobian_true_nonsymmetric_preconditioner"
+)
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if runcheck == 1:
+    passed = True
+    for i in range(11):
+        timestep = str(i).rjust(5, "0")
+        filename = f"/{run_name}.out.press.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Pressure for timestep {timestep}",
+        ):
+            passed = False
+        filename = f"/{run_name}.out.satur.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Saturation for timestep {timestep}",
+        ):
+            passed = False
+    if passed:
+        print(f"{run_name} : PASSED")
+    else:
+        print(f"{run_name} : FAILED")
+        sys.exit(1)
diff --git a/test/python/overland_tiltedV_KWE.py b/test/python/overland_tiltedV_KWE.py
index 91ca79992..2eeae05b7 100644
--- a/test/python/overland_tiltedV_KWE.py
+++ b/test/python/overland_tiltedV_KWE.py
@@ -1,13 +1,15 @@
-#-----------------------------------------------------------------------------
-#running different configuraitons of tilted V
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+# running different configuraitons of tilted V
+# -----------------------------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
 overland = Run("overland_tiltedV_KWE", __file__)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 overland.FileVersion = 4
 
@@ -15,9 +17,9 @@
 overland.Process.Topology.Q = 1
 overland.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.ComputationalGrid.Lower.X = 0.0
 overland.ComputationalGrid.Lower.Y = 0.0
@@ -29,27 +31,27 @@
 
 overland.ComputationalGrid.DX = 10.0
 overland.ComputationalGrid.DY = 10.0
-overland.ComputationalGrid.DZ = .05
+overland.ComputationalGrid.DZ = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland.GeomInput.Names = 'domaininput leftinput rightinput channelinput'
+overland.GeomInput.Names = "domaininput leftinput rightinput channelinput"
 
-overland.GeomInput.domaininput.GeomName = 'domain'
-overland.GeomInput.leftinput.GeomName = 'left'
-overland.GeomInput.rightinput.GeomName = 'right'
-overland.GeomInput.channelinput.GeomName = 'channel'
+overland.GeomInput.domaininput.GeomName = "domain"
+overland.GeomInput.leftinput.GeomName = "left"
+overland.GeomInput.rightinput.GeomName = "right"
+overland.GeomInput.channelinput.GeomName = "channel"
 
-overland.GeomInput.domaininput.InputType = 'Box'
-overland.GeomInput.leftinput.InputType = 'Box'
-overland.GeomInput.rightinput.InputType = 'Box'
-overland.GeomInput.channelinput.InputType = 'Box'
+overland.GeomInput.domaininput.InputType = "Box"
+overland.GeomInput.leftinput.InputType = "Box"
+overland.GeomInput.rightinput.InputType = "Box"
+overland.GeomInput.channelinput.InputType = "Box"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.Geom.domain.Lower.X = 0.0
 overland.Geom.domain.Lower.Y = 0.0
@@ -58,11 +60,11 @@
 overland.Geom.domain.Upper.X = 50.0
 overland.Geom.domain.Upper.Y = 50.0
 overland.Geom.domain.Upper.Z = 0.05
-overland.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+overland.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Left Slope Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.Geom.left.Lower.X = 0.0
 overland.Geom.left.Lower.Y = 0.0
@@ -72,9 +74,9 @@
 overland.Geom.left.Upper.Y = 50.0
 overland.Geom.left.Upper.Z = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Right Slope Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.Geom.right.Lower.X = 30.0
 overland.Geom.right.Lower.Y = 0.0
@@ -84,9 +86,9 @@
 overland.Geom.right.Upper.Y = 50.0
 overland.Geom.right.Upper.Z = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Channel Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 overland.Geom.channel.Lower.X = 20.0
 overland.Geom.channel.Lower.Y = 0.0
@@ -96,207 +98,207 @@
 overland.Geom.channel.Upper.Y = 50.0
 overland.Geom.channel.Upper.Z = 0.05
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Geom.Perm.Names = 'domain'
-overland.Geom.domain.Perm.Type = 'Constant'
+overland.Geom.Perm.Names = "domain"
+overland.Geom.domain.Perm.Type = "Constant"
 overland.Geom.domain.Perm.Value = 0.0000001
 
-overland.Perm.TensorType = 'TensorByGeom'
+overland.Perm.TensorType = "TensorByGeom"
 
-overland.Geom.Perm.TensorByGeom.Names = 'domain'
+overland.Geom.Perm.TensorByGeom.Names = "domain"
 
 overland.Geom.domain.Perm.TensorValX = 1.0
 overland.Geom.domain.Perm.TensorValY = 1.0
 overland.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.SpecificStorage.Type = 'Constant'
-overland.SpecificStorage.GeomNames = 'domain'
+overland.SpecificStorage.Type = "Constant"
+overland.SpecificStorage.GeomNames = "domain"
 overland.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Phase.Names = 'water'
+overland.Phase.Names = "water"
 
-overland.Phase.water.Density.Type = 'Constant'
+overland.Phase.water.Density.Type = "Constant"
 overland.Phase.water.Density.Value = 1.0
 
-overland.Phase.water.Viscosity.Type = 'Constant'
+overland.Phase.water.Viscosity.Type = "Constant"
 overland.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Contaminants.Names = ''
+overland.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Geom.Retardation.GeomNames = ''
+overland.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 overland.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 overland.TimingInfo.BaseUnit = 0.05
 overland.TimingInfo.StartCount = 0
 overland.TimingInfo.StartTime = 0.0
 overland.TimingInfo.StopTime = 2.0
 overland.TimingInfo.DumpInterval = -2
-overland.TimeStep.Type = 'Constant'
+overland.TimeStep.Type = "Constant"
 overland.TimeStep.Value = 0.05
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Geom.Porosity.GeomNames = 'domain'
-overland.Geom.domain.Porosity.Type = 'Constant'
+overland.Geom.Porosity.GeomNames = "domain"
+overland.Geom.domain.Porosity.Type = "Constant"
 overland.Geom.domain.Porosity.Value = 0.01
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Domain.GeomName = 'domain'
+overland.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Phase.RelPerm.Type = 'VanGenuchten'
-overland.Phase.RelPerm.GeomNames = 'domain'
+overland.Phase.RelPerm.Type = "VanGenuchten"
+overland.Phase.RelPerm.GeomNames = "domain"
 
 overland.Geom.domain.RelPerm.Alpha = 6.0
-overland.Geom.domain.RelPerm.N = 2.
+overland.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland.Phase.Saturation.Type = 'VanGenuchten'
-overland.Phase.Saturation.GeomNames = 'domain'
+overland.Phase.Saturation.Type = "VanGenuchten"
+overland.Phase.Saturation.GeomNames = "domain"
 
 overland.Geom.domain.Saturation.Alpha = 6.0
-overland.Geom.domain.Saturation.N = 2.
+overland.Geom.domain.Saturation.N = 2.0
 overland.Geom.domain.Saturation.SRes = 0.2
 overland.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Wells.Names = ''
+overland.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Cycle.Names = 'constant rainrec'
-overland.Cycle.constant.Names = 'alltime'
+overland.Cycle.Names = "constant rainrec"
+overland.Cycle.constant.Names = "alltime"
 overland.Cycle.constant.alltime.Length = 1
 overland.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-overland.Cycle.rainrec.Names = 'rain rec'
+overland.Cycle.rainrec.Names = "rain rec"
 overland.Cycle.rainrec.rain.Length = 2
 overland.Cycle.rainrec.rec.Length = 300
 overland.Cycle.rainrec.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 overland.BCPressure.PatchNames = overland.Geom.domain.Patches
 
-overland.Patch.x_lower.BCPressure.Type = 'FluxConst'
-overland.Patch.x_lower.BCPressure.Cycle = 'constant'
+overland.Patch.x_lower.BCPressure.Type = "FluxConst"
+overland.Patch.x_lower.BCPressure.Cycle = "constant"
 overland.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-overland.Patch.y_lower.BCPressure.Type = 'FluxConst'
-overland.Patch.y_lower.BCPressure.Cycle = 'constant'
+overland.Patch.y_lower.BCPressure.Type = "FluxConst"
+overland.Patch.y_lower.BCPressure.Cycle = "constant"
 overland.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-overland.Patch.z_lower.BCPressure.Type = 'FluxConst'
-overland.Patch.z_lower.BCPressure.Cycle = 'constant'
+overland.Patch.z_lower.BCPressure.Type = "FluxConst"
+overland.Patch.z_lower.BCPressure.Cycle = "constant"
 overland.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-overland.Patch.x_upper.BCPressure.Type = 'FluxConst'
-overland.Patch.x_upper.BCPressure.Cycle = 'constant'
+overland.Patch.x_upper.BCPressure.Type = "FluxConst"
+overland.Patch.x_upper.BCPressure.Cycle = "constant"
 overland.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-overland.Patch.y_upper.BCPressure.Type = 'FluxConst'
-overland.Patch.y_upper.BCPressure.Cycle = 'constant'
+overland.Patch.y_upper.BCPressure.Type = "FluxConst"
+overland.Patch.y_upper.BCPressure.Cycle = "constant"
 overland.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
 ## overland flow boundary condition with very heavy rainfall
-overland.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-overland.Patch.z_upper.BCPressure.Cycle = 'rainrec'
+overland.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+overland.Patch.z_upper.BCPressure.Cycle = "rainrec"
 overland.Patch.z_upper.BCPressure.rain.Value = -0.01
 overland.Patch.z_upper.BCPressure.rec.Value = 0.0000
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-overland.Mannings.Type = 'Constant'
-overland.Mannings.GeomNames = 'domain'
-overland.Mannings.Geom.domain.Value = 3.e-6
+overland.Mannings.Type = "Constant"
+overland.Mannings.GeomNames = "domain"
+overland.Mannings.Geom.domain.Value = 3.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.PhaseSources.water.Type = 'Constant'
-overland.PhaseSources.water.GeomNames = 'domain'
+overland.PhaseSources.water.Type = "Constant"
+overland.PhaseSources.water.GeomNames = "domain"
 overland.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.KnownSolution = 'NoKnownSolution'
+overland.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.Solver = 'Richards'
+overland.Solver = "Richards"
 overland.Solver.MaxIter = 2500
 
 overland.Solver.Nonlinear.MaxIter = 100
 overland.Solver.Nonlinear.ResidualTol = 1e-9
-overland.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+overland.Solver.Nonlinear.EtaChoice = "EtaConstant"
 overland.Solver.Nonlinear.EtaValue = 0.01
 overland.Solver.Nonlinear.UseJacobian = False
 overland.Solver.Nonlinear.DerivativeEpsilon = 1e-15
 overland.Solver.Nonlinear.StepTol = 1e-20
-overland.Solver.Nonlinear.Globalization = 'LineSearch'
+overland.Solver.Nonlinear.Globalization = "LineSearch"
 overland.Solver.Linear.KrylovDimension = 50
 overland.Solver.Linear.MaxRestart = 2
-overland.Solver.OverlandKinematic.Epsilon = 1E-5
+overland.Solver.OverlandKinematic.Epsilon = 1e-5
 
-overland.Solver.Linear.Preconditioner = 'PFMG'
+overland.Solver.Linear.Preconditioner = "PFMG"
 overland.Solver.PrintSubsurf = False
-overland.Solver.Drop = 1E-20
-overland.Solver.AbsTol = 1E-10
+overland.Solver.Drop = 1e-20
+overland.Solver.AbsTol = 1e-10
 
 overland.Solver.WriteSiloSubsurfData = False
 overland.Solver.WriteSiloPressure = False
@@ -305,218 +307,276 @@
 overland.Solver.WriteSiloSaturation = False
 overland.Solver.WriteSiloConcentration = False
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be at the bottom of the domain, the top layer is initially dry
-overland.ICPressure.Type = 'HydroStaticPatch'
-overland.ICPressure.GeomNames = 'domain'
+overland.ICPressure.Type = "HydroStaticPatch"
+overland.ICPressure.GeomNames = "domain"
 overland.Geom.domain.ICPressure.Value = -3.0
 
-overland.Geom.domain.ICPressure.RefGeom = 'domain'
-overland.Geom.domain.ICPressure.RefPatch = 'z_upper'
+overland.Geom.domain.ICPressure.RefGeom = "domain"
+overland.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+
+# set runcheck to 1 if you want to run the pass fail tests
+runcheck = 1
+correct_output_dir_name = get_absolute_path("../correct_output")
+
+
+# -----------------------------------------------------------------------------
 # Original formulation with a zero value channel
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-overland.TopoSlopesX.Type = 'Constant'
-overland.TopoSlopesX.GeomNames = 'left right channel'
+overland.TopoSlopesX.Type = "Constant"
+overland.TopoSlopesX.GeomNames = "left right channel"
 overland.TopoSlopesX.Geom.left.Value = -0.01
 overland.TopoSlopesX.Geom.right.Value = 0.01
 overland.TopoSlopesX.Geom.channel.Value = 0.00
 
-overland.TopoSlopesY.Type = 'Constant'
-overland.TopoSlopesY.GeomNames = 'domain'
+overland.TopoSlopesY.Type = "Constant"
+overland.TopoSlopesY.GeomNames = "domain"
 overland.TopoSlopesY.Geom.domain.Value = 0.01
 
-#original approach from K&M AWR 2006
-overland.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
+# original approach from K&M AWR 2006
+overland.Patch.z_upper.BCPressure.Type = "OverlandFlow"
 overland.Solver.Nonlinear.UseJacobian = False
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'PFSymmetric'
-
-# Commented lines are from original TCL test - will need to convert to Python if running with
-# Python pftools
-
-# set runname TiltedV_Overland
-# puts "##########"
-# puts $runname
-# pfrun $runname
-# pfundist $runname
-# if $runcheck==1 {
-#   set passed 1
-#   foreach i "00000 00001 00002 00003 00004 00005 00006 00007 00008 00009 00010" {
-#     if ![pftestFile $runname.out.press.$i.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#     if ![pftestFile  $runname.out.satur.$i.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#   }
-#   if $passed {
-#     puts "$runname : PASSED"
-#   } {
-#     puts "$runname : FAILED"
-#   }
-# }
-
-#original approach from K&M AWR 2006 with analytical jacobian
-overland.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
+overland.Solver.Linear.Preconditioner.PCMatrixType = "PFSymmetric"
+
+run_name = "TiltedV_Overland"
+overland.set_name(run_name)
+print("##########")
+print(run_name)
+new_output_dir_name = get_absolute_path("test_output/" + run_name)
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if runcheck == 1:
+    passed = True
+    for i in range(11):
+        timestep = str(i).rjust(5, "0")
+        filename = f"/{run_name}.out.press.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Pressure for timestep {timestep}",
+        ):
+            passed = False
+        filename = f"/{run_name}.out.satur.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Saturation for timestep {timestep}",
+        ):
+            passed = False
+
+    if passed:
+        print(f"{run_name} : PASSED")
+    else:
+        print(f"{run_name} : FAILED")
+        sys.exit(1)
+
+
+# original approach from K&M AWR 2006 with analytical jacobian
+overland.Patch.z_upper.BCPressure.Type = "OverlandFlow"
 overland.Solver.Nonlinear.UseJacobian = True
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'PFSymmetric'
-
-# set runname TiltedV_Overland
-# puts "##########"
-# puts "Running $runname Jacobian True"
-# pfrun $runname
-# pfundist $runname
-# if $runcheck==1 {
-#   set passed 1
-#   foreach i "00000 00001 00002 00003 00004 00005 00006 00007 00008 00009 00010" {
-#     if ![pftestFile $runname.out.press.$i.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#     if ![pftestFile  $runname.out.satur.$i.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#   }
-#   if $passed {
-#     puts "$runname : PASSED"
-#   } {
-#     puts "$runname : FAILED"
-#   }
-# }
-
-#original approach from K&M AWR 2006 with analytical jacobian and nonsymmetric preconditioner
-overland.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
+overland.Solver.Linear.Preconditioner.PCMatrixType = "PFSymmetric"
+
+
+print("##########")
+print(f"Running {run_name} Jacobian True")
+new_output_dir_name = get_absolute_path("test_output/" + f"{run_name}_jacobian_true")
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if runcheck == 1:
+    passed = True
+    for i in range(11):
+        timestep = str(i).rjust(5, "0")
+        filename = f"/{run_name}.out.press.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Pressure for timestep {timestep}",
+        ):
+            passed = False
+        filename = f"/{run_name}.out.satur.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Saturation for timestep {timestep}",
+        ):
+            passed = False
+
+    if passed:
+        print(f"{run_name} : PASSED")
+    else:
+        print(f"{run_name} : FAILED")
+        sys.exit(1)
+
+
+# original approach from K&M AWR 2006 with analytical jacobian and nonsymmetric preconditioner
+overland.Patch.z_upper.BCPressure.Type = "OverlandFlow"
 overland.Solver.Nonlinear.UseJacobian = True
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'FullJacobian'
-
-# set runname TiltedV_Overland
-# puts "##########"
-# puts "Running $runname Jacobian True Nonsymmetric Preconditioner"
-# pfrun $runname
-# pfundist $runname
-# if $runcheck==1 {
-#   set passed 1
-#   foreach i "00000 00001 00002 00003 00004 00005 00006 00007 00008 00009 00010" {
-#     if ![pftestFile $runname.out.press.$i.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#     if ![pftestFile  $runname.out.satur.$i.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#   }
-#   if $passed {
-#     puts "$runname : PASSED"
-#   } {
-#     puts "$runname : FAILED"
-#   }
-# }
-
-#-----------------------------------------------------------------------------
+overland.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
+
+print("##########")
+print(f"Running {run_name} Jacobian True Nonsymmetric Preconditioner")
+new_output_dir_name = get_absolute_path(
+    "test_output/" + f"{run_name}_jacobian_true_nonsymmetric_preconditioner"
+)
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if runcheck == 1:
+    passed = True
+    for i in range(11):
+        timestep = str(i).rjust(5, "0")
+        filename = f"/{run_name}.out.press.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Pressure for timestep {timestep}",
+        ):
+            passed = False
+        filename = f"/{run_name}.out.satur.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Saturation for timestep {timestep}",
+        ):
+            passed = False
+
+    if passed:
+        print(f"{run_name} : PASSED")
+    else:
+        print(f"{run_name} : FAILED")
+        sys.exit(1)
+
+
+# -----------------------------------------------------------------------------
 # New kinematic formulations without the zero channel
 # Note: The difference in configuration here is to be consistent with the way
 #   the upwinding is handled for the new and original fomulations.
 #   These two results should be almost identiacl for the new and old formulations
-#-----------------------------------------------------------------------------
-overland.TopoSlopesX.Type = 'Constant'
-overland.TopoSlopesX.GeomNames = 'left right channel'
+# -----------------------------------------------------------------------------
+overland.TopoSlopesX.Type = "Constant"
+overland.TopoSlopesX.GeomNames = "left right channel"
 overland.TopoSlopesX.Geom.left.Value = -0.01
 overland.TopoSlopesX.Geom.right.Value = 0.01
 overland.TopoSlopesX.Geom.channel.Value = 0.01
 
-overland.TopoSlopesY.Type = 'Constant'
-overland.TopoSlopesY.GeomNames = 'domain'
+overland.TopoSlopesY.Type = "Constant"
+overland.TopoSlopesY.GeomNames = "domain"
 overland.TopoSlopesY.Geom.domain.Value = 0.01
 
 # run with KWE upwinding
-overland.Patch.z_upper.BCPressure.Type = 'OverlandKinematic'
+overland.Patch.z_upper.BCPressure.Type = "OverlandKinematic"
 overland.Solver.Nonlinear.UseJacobian = False
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'PFSymmetric'
-
-# set runname TiltedV_OverlandKin
-# puts "##########"
-# puts "Running $runname"
-# pfrun $runname
-# pfundist $runname
-# if $runcheck==1 {
-#   set passed 1
-#   foreach i "00000 00001 00002 00003 00004 00005 00006 00007 00008 00009 00010" {
-#     if ![pftestFile $runname.out.press.$i.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#     if ![pftestFile  $runname.out.satur.$i.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#   }
-#   if $passed {
-#     puts "$runname : PASSED"
-#   } {
-#     puts "$runname : FAILED"
-#   }
-# }
+overland.Solver.Linear.Preconditioner.PCMatrixType = "PFSymmetric"
+
+
+run_name = "TiltedV_OverlandKin"
+overland.set_name(run_name)
+print("##########")
+print(f"Running {run_name}")
+new_output_dir_name = get_absolute_path("test_output/" + f"{run_name}")
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if runcheck == 1:
+    passed = True
+    for i in range(11):
+        timestep = str(i).rjust(5, "0")
+        filename = f"/{run_name}.out.press.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Pressure for timestep {timestep}",
+        ):
+            passed = False
+        filename = f"/{run_name}.out.satur.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Saturation for timestep {timestep}",
+        ):
+            passed = False
+
+    if passed:
+        print(f"{run_name} : PASSED")
+    else:
+        print(f"{run_name} : FAILED")
+        sys.exit(1)
+
 
 # run with KWE upwinding and analytical jacobian
-overland.Patch.z_upper.BCPressure.Type = 'OverlandKinematic'
+overland.Patch.z_upper.BCPressure.Type = "OverlandKinematic"
 overland.Solver.Nonlinear.UseJacobian = True
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'PFSymmetric'
-
-# set runname TiltedV_OverlandKin
-# puts "##########"
-# puts "Running $runname Jacobian True"
-# pfrun $runname
-# pfundist $runname
-# if $runcheck==1 {
-#   set passed 1
-#   foreach i "00000 00001 00002 00003 00004 00005 00006 00007 00008 00009 00010" {
-#     if ![pftestFile $runname.out.press.$i.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#     if ![pftestFile  $runname.out.satur.$i.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#   }
-#   if $passed {
-#     puts "$runname : PASSED"
-#   } {
-#     puts "$runname : FAILED"
-#   }
-# }
+overland.Solver.Linear.Preconditioner.PCMatrixType = "PFSymmetric"
+
+print("##########")
+print(f"Running {run_name} Jacobian True")
+new_output_dir_name = get_absolute_path("test_output/" + f"{run_name}_jacobian_true")
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if runcheck == 1:
+    passed = True
+    for i in range(11):
+        timestep = str(i).rjust(5, "0")
+        filename = f"/{run_name}.out.press.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Pressure for timestep {timestep}",
+        ):
+            passed = False
+        filename = f"/{run_name}.out.satur.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Saturation for timestep {timestep}",
+        ):
+            passed = False
+
+    if passed:
+        print(f"{run_name} : PASSED")
+    else:
+        print(f"{run_name} : FAILED")
+        sys.exit(1)
+
 
 # run with KWE upwinding and analytical jacobian and nonsymmetric preconditioner
-overland.Patch.z_upper.BCPressure.Type = 'OverlandKinematic'
+overland.Patch.z_upper.BCPressure.Type = "OverlandKinematic"
 overland.Solver.Nonlinear.UseJacobian = True
-overland.Solver.Linear.Preconditioner.PCMatrixType = 'FullJacobian'
-
-# set runname TiltedV_OverlandKin
-# puts "##########"
-# puts "Running $runname Jacobian True Nonsymmetric Preconditioner"
-# pfrun $runname
-# pfundist $runname
-# if $runcheck==1 {
-#   set passed 1
-#   foreach i "00000 00001 00002 00003 00004 00005 00006 00007 00008 00009 00010" {
-#     if ![pftestFile $runname.out.press.$i.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#     if ![pftestFile  $runname.out.satur.$i.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
-#       set passed 0
-#     }
-#   }
-#   if $passed {
-#     puts "$runname : PASSED"
-#   } {
-#     puts "$runname : FAILED"
-#   }
-# }
-
-#-----------------------------------------------------------------------------
-# Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/ov_kwe')
-mkdir(dir_name)
-overland.run(working_directory=dir_name)
-
+overland.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
+
+
+print("##########")
+print(f"Running {run_name} Jacobian True Nonsymmetric Preconditioner")
+new_output_dir_name = get_absolute_path(
+    "test_output/" + f"{run_name}_jacobian_true_nonsymmetric_preconditioner"
+)
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if runcheck == 1:
+    passed = True
+    for i in range(11):
+        timestep = str(i).rjust(5, "0")
+        filename = f"/{run_name}.out.press.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Pressure for timestep {timestep}",
+        ):
+            passed = False
+        filename = f"/{run_name}.out.satur.{timestep}.pfb"
+        if not pf_test_file(
+            new_output_dir_name + filename,
+            correct_output_dir_name + filename,
+            f"Max difference in Saturation for timestep {timestep}",
+        ):
+            passed = False
+
+    if passed:
+        print(f"{run_name} : PASSED")
+    else:
+        print(f"{run_name} : FAILED")
+        sys.exit(1)
diff --git a/test/python/overland_tiltedV_MGSemiTest.py b/test/python/overland_tiltedV_MGSemiTest.py
new file mode 100644
index 000000000..583a85cbe
--- /dev/null
+++ b/test/python/overland_tiltedV_MGSemiTest.py
@@ -0,0 +1,503 @@
+# -----------------------------------------------------------------------------
+# MGSemi preconditioner test for overland flow with tilted V
+# running different configurations of tilted V, OverlandKinematic, OverlandFlow and OverlandDiffusive
+# with and without analytical jacobian and with MGSemi preconditioner
+# The results should be almost identical for the different configurations
+# The test is to check the correctness of the MGSemi preconditioner
+# RMM, 2024-07-02
+# -----------------------------------------------------------------------------
+
+import sys
+from parflow import Run
+from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file, pf_test_file_with_abs
+import os
+
+
+def check_output(run_name, correct_root, correct_output_dir_name):
+
+    passed = True
+    sig_digits = 4
+    abs_value = 1e-12
+    test_files = ["press"]
+    for timestep in range(20):
+        for test_file in test_files:
+            filename = f"/{run_name}.out.{test_file}.{timestep:05d}.pfb"
+            correct_filename = f"/{correct_root}.out.{test_file}.{timestep:05d}.pfb"
+            result = pf_test_file_with_abs(
+                new_output_dir_name + filename,
+                correct_output_dir_name + correct_filename,
+                f"Max difference in {new_output_dir_name + filename}",
+                abs_value,
+                sig_digits,
+            )
+            if not result:
+                passed = False
+
+    return passed
+
+
+overland = Run("overland_tiltedV_KWE", __file__)
+
+# -----------------------------------------------------------------------------
+
+overland.FileVersion = 4
+
+overland.Process.Topology.P = 1
+overland.Process.Topology.Q = 1
+overland.Process.Topology.R = 1
+
+# ---------------------------------------------------------
+# Computational Grid
+# ---------------------------------------------------------
+
+overland.ComputationalGrid.Lower.X = 0.0
+overland.ComputationalGrid.Lower.Y = 0.0
+overland.ComputationalGrid.Lower.Z = 0.0
+
+overland.ComputationalGrid.NX = 5
+overland.ComputationalGrid.NY = 5
+overland.ComputationalGrid.NZ = 1
+
+overland.ComputationalGrid.DX = 10.0
+overland.ComputationalGrid.DY = 10.0
+overland.ComputationalGrid.DZ = 0.05
+
+# ---------------------------------------------------------
+# The Names of the GeomInputs
+# ---------------------------------------------------------
+
+overland.GeomInput.Names = "domaininput leftinput rightinput channelinput"
+
+overland.GeomInput.domaininput.GeomName = "domain"
+overland.GeomInput.leftinput.GeomName = "left"
+overland.GeomInput.rightinput.GeomName = "right"
+overland.GeomInput.channelinput.GeomName = "channel"
+
+overland.GeomInput.domaininput.InputType = "Box"
+overland.GeomInput.leftinput.InputType = "Box"
+overland.GeomInput.rightinput.InputType = "Box"
+overland.GeomInput.channelinput.InputType = "Box"
+
+# ---------------------------------------------------------
+# Domain Geometry
+# ---------------------------------------------------------
+
+overland.Geom.domain.Lower.X = 0.0
+overland.Geom.domain.Lower.Y = 0.0
+overland.Geom.domain.Lower.Z = 0.0
+
+overland.Geom.domain.Upper.X = 50.0
+overland.Geom.domain.Upper.Y = 50.0
+overland.Geom.domain.Upper.Z = 0.05
+overland.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
+
+# ---------------------------------------------------------
+# Left Slope Geometry
+# ---------------------------------------------------------
+
+overland.Geom.left.Lower.X = 0.0
+overland.Geom.left.Lower.Y = 0.0
+overland.Geom.left.Lower.Z = 0.0
+
+overland.Geom.left.Upper.X = 20.0
+overland.Geom.left.Upper.Y = 50.0
+overland.Geom.left.Upper.Z = 0.05
+
+# ---------------------------------------------------------
+# Right Slope Geometry
+# ---------------------------------------------------------
+
+overland.Geom.right.Lower.X = 30.0
+overland.Geom.right.Lower.Y = 0.0
+overland.Geom.right.Lower.Z = 0.0
+
+overland.Geom.right.Upper.X = 50.0
+overland.Geom.right.Upper.Y = 50.0
+overland.Geom.right.Upper.Z = 0.05
+
+# ---------------------------------------------------------
+# Channel Geometry
+# ---------------------------------------------------------
+
+overland.Geom.channel.Lower.X = 20.0
+overland.Geom.channel.Lower.Y = 0.0
+overland.Geom.channel.Lower.Z = 0.0
+
+overland.Geom.channel.Upper.X = 30.0
+overland.Geom.channel.Upper.Y = 50.0
+overland.Geom.channel.Upper.Z = 0.05
+
+# -----------------------------------------------------------------------------
+# Perm
+# -----------------------------------------------------------------------------
+
+overland.Geom.Perm.Names = "domain"
+overland.Geom.domain.Perm.Type = "Constant"
+overland.Geom.domain.Perm.Value = 0.0000001
+
+overland.Perm.TensorType = "TensorByGeom"
+
+overland.Geom.Perm.TensorByGeom.Names = "domain"
+
+overland.Geom.domain.Perm.TensorValX = 1.0
+overland.Geom.domain.Perm.TensorValY = 1.0
+overland.Geom.domain.Perm.TensorValZ = 1.0
+
+# -----------------------------------------------------------------------------
+# Specific Storage
+# -----------------------------------------------------------------------------
+
+overland.SpecificStorage.Type = "Constant"
+overland.SpecificStorage.GeomNames = "domain"
+overland.Geom.domain.SpecificStorage.Value = 1.0e-4
+
+# -----------------------------------------------------------------------------
+# Phases
+# -----------------------------------------------------------------------------
+
+overland.Phase.Names = "water"
+
+overland.Phase.water.Density.Type = "Constant"
+overland.Phase.water.Density.Value = 1.0
+
+overland.Phase.water.Viscosity.Type = "Constant"
+overland.Phase.water.Viscosity.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Contaminants
+# -----------------------------------------------------------------------------
+
+overland.Contaminants.Names = ""
+
+# -----------------------------------------------------------------------------
+# Retardation
+# -----------------------------------------------------------------------------
+
+overland.Geom.Retardation.GeomNames = ""
+
+# -----------------------------------------------------------------------------
+# Gravity
+# -----------------------------------------------------------------------------
+
+overland.Gravity = 1.0
+
+# -----------------------------------------------------------------------------
+# Setup timing info
+# -----------------------------------------------------------------------------
+overland.TimingInfo.BaseUnit = 0.05
+overland.TimingInfo.StartCount = 0
+overland.TimingInfo.StartTime = 0.0
+overland.TimingInfo.StopTime = 2.0
+overland.TimingInfo.DumpInterval = -2
+overland.TimeStep.Type = "Constant"
+overland.TimeStep.Value = 0.05
+
+# -----------------------------------------------------------------------------
+# Porosity
+# -----------------------------------------------------------------------------
+
+overland.Geom.Porosity.GeomNames = "domain"
+overland.Geom.domain.Porosity.Type = "Constant"
+overland.Geom.domain.Porosity.Value = 0.01
+
+# -----------------------------------------------------------------------------
+# Domain
+# -----------------------------------------------------------------------------
+
+overland.Domain.GeomName = "domain"
+
+# -----------------------------------------------------------------------------
+# Relative Permeability
+# -----------------------------------------------------------------------------
+
+overland.Phase.RelPerm.Type = "VanGenuchten"
+overland.Phase.RelPerm.GeomNames = "domain"
+
+overland.Geom.domain.RelPerm.Alpha = 6.0
+overland.Geom.domain.RelPerm.N = 2.0
+
+# ---------------------------------------------------------
+# Saturation
+# ---------------------------------------------------------
+
+overland.Phase.Saturation.Type = "VanGenuchten"
+overland.Phase.Saturation.GeomNames = "domain"
+
+overland.Geom.domain.Saturation.Alpha = 6.0
+overland.Geom.domain.Saturation.N = 2.0
+overland.Geom.domain.Saturation.SRes = 0.2
+overland.Geom.domain.Saturation.SSat = 1.0
+
+# -----------------------------------------------------------------------------
+# Wells
+# -----------------------------------------------------------------------------
+
+overland.Wells.Names = ""
+
+# -----------------------------------------------------------------------------
+# Time Cycles
+# -----------------------------------------------------------------------------
+
+overland.Cycle.Names = "constant rainrec"
+overland.Cycle.constant.Names = "alltime"
+overland.Cycle.constant.alltime.Length = 1
+overland.Cycle.constant.Repeat = -1
+
+# rainfall and recession time periods are defined here
+# rain for 1 hour, recession for 2 hours
+
+overland.Cycle.rainrec.Names = "rain rec"
+overland.Cycle.rainrec.rain.Length = 2
+overland.Cycle.rainrec.rec.Length = 300
+overland.Cycle.rainrec.Repeat = -1
+
+# -----------------------------------------------------------------------------
+# Boundary Conditions: Pressure
+# -----------------------------------------------------------------------------
+
+overland.BCPressure.PatchNames = overland.Geom.domain.Patches
+
+overland.Patch.x_lower.BCPressure.Type = "FluxConst"
+overland.Patch.x_lower.BCPressure.Cycle = "constant"
+overland.Patch.x_lower.BCPressure.alltime.Value = 0.0
+
+overland.Patch.y_lower.BCPressure.Type = "FluxConst"
+overland.Patch.y_lower.BCPressure.Cycle = "constant"
+overland.Patch.y_lower.BCPressure.alltime.Value = 0.0
+
+overland.Patch.z_lower.BCPressure.Type = "FluxConst"
+overland.Patch.z_lower.BCPressure.Cycle = "constant"
+overland.Patch.z_lower.BCPressure.alltime.Value = 0.0
+
+overland.Patch.x_upper.BCPressure.Type = "FluxConst"
+overland.Patch.x_upper.BCPressure.Cycle = "constant"
+overland.Patch.x_upper.BCPressure.alltime.Value = 0.0
+
+overland.Patch.y_upper.BCPressure.Type = "FluxConst"
+overland.Patch.y_upper.BCPressure.Cycle = "constant"
+overland.Patch.y_upper.BCPressure.alltime.Value = 0.0
+
+## overland flow boundary condition with very heavy rainfall
+overland.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+overland.Patch.z_upper.BCPressure.Cycle = "rainrec"
+overland.Patch.z_upper.BCPressure.rain.Value = -0.01
+overland.Patch.z_upper.BCPressure.rec.Value = 0.0000
+
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
+
+overland.Mannings.Type = "Constant"
+overland.Mannings.GeomNames = "domain"
+overland.Mannings.Geom.domain.Value = 3.0e-6
+
+# -----------------------------------------------------------------------------
+# Phase sources:
+# -----------------------------------------------------------------------------
+
+overland.PhaseSources.water.Type = "Constant"
+overland.PhaseSources.water.GeomNames = "domain"
+overland.PhaseSources.water.Geom.domain.Value = 0.0
+
+# -----------------------------------------------------------------------------
+# Exact solution specification for error calculations
+# -----------------------------------------------------------------------------
+
+overland.KnownSolution = "NoKnownSolution"
+
+# -----------------------------------------------------------------------------
+# Set solver parameters
+# -----------------------------------------------------------------------------
+
+overland.Solver = "Richards"
+overland.Solver.MaxIter = 2500
+
+overland.Solver.Nonlinear.MaxIter = 100
+overland.Solver.Nonlinear.ResidualTol = 1e-9
+overland.Solver.Nonlinear.EtaChoice = "EtaConstant"
+overland.Solver.Nonlinear.EtaValue = 0.01
+overland.Solver.Nonlinear.UseJacobian = False
+overland.Solver.Nonlinear.DerivativeEpsilon = 1e-15
+overland.Solver.Nonlinear.StepTol = 1e-20
+overland.Solver.Nonlinear.Globalization = "LineSearch"
+overland.Solver.Linear.KrylovDimension = 50
+overland.Solver.Linear.MaxRestart = 2
+overland.Solver.OverlandKinematic.Epsilon = 1e-5
+
+overland.Solver.Linear.Preconditioner = "MGSemi"
+overland.Solver.PrintSubsurf = False
+overland.Solver.Drop = 1e-20
+overland.Solver.AbsTol = 1e-10
+
+overland.Solver.WriteSiloSubsurfData = False
+overland.Solver.WriteSiloPressure = False
+overland.Solver.WriteSiloSlopes = False
+
+overland.Solver.WriteSiloSaturation = False
+overland.Solver.WriteSiloConcentration = False
+
+# ---------------------------------------------------------
+# Initial conditions: water pressure
+# ---------------------------------------------------------
+
+# set water table to be at the bottom of the domain, the top layer is initially dry
+overland.ICPressure.Type = "HydroStaticPatch"
+overland.ICPressure.GeomNames = "domain"
+overland.Geom.domain.ICPressure.Value = -3.0
+
+overland.Geom.domain.ICPressure.RefGeom = "domain"
+overland.Geom.domain.ICPressure.RefPatch = "z_upper"
+
+
+# -----------------------------------------------------------------------------
+# New kinematic formulations without the zero channel
+# Note: The difference in configuration here is to be consistent with the way
+# the upwinding is handled for the new and original formulations.
+# These two results should be almost identical for the new and old formulations
+# -----------------------------------------------------------------------------
+overland.TopoSlopesX.Type = "Constant"
+overland.TopoSlopesX.GeomNames = "left right channel"
+overland.TopoSlopesX.Geom.left.Value = -0.01
+overland.TopoSlopesX.Geom.right.Value = 0.01
+overland.TopoSlopesX.Geom.channel.Value = 0.01
+
+overland.TopoSlopesY.Type = "Constant"
+overland.TopoSlopesY.GeomNames = "domain"
+overland.TopoSlopesY.Geom.domain.Value = 0.01
+
+# run with KWE upwinding
+overland.Patch.z_upper.BCPressure.Type = "OverlandKinematic"
+overland.Solver.Nonlinear.UseJacobian = False
+overland.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
+
+passed = True
+
+correct_output_dir_name = get_absolute_path(
+    "../correct_output/TiltedV_OverlandKin_MGSemi"
+)
+correct_root = "TiltedV_OverlandKin"
+
+run_name = "TiltedV_OverlandKin_JacFalse_MGSemi"
+overland.set_name(run_name)
+print("##########")
+print(f"Running {run_name}")
+test_dir_name = get_absolute_path("test_output/")
+new_output_dir_name = os.path.join(test_dir_name, f"{run_name}")
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if not check_output(run_name, correct_root, correct_output_dir_name):
+    passed = False
+
+run_name = "TiltedV_OverlandKin_JacTrue_MGSemi"
+overland.set_name(run_name)
+# run with KWE upwinding and analytical jacobian
+overland.Patch.z_upper.BCPressure.Type = "OverlandKinematic"
+overland.Solver.Nonlinear.UseJacobian = True
+overland.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
+
+print("##########")
+print(f"Running {run_name} Jacobian True")
+new_output_dir_name = os.path.join(test_dir_name, f"{run_name}")
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if not check_output(run_name, correct_root, correct_output_dir_name):
+    passed = False
+
+
+# -----------------------------------------------------------------------------
+# Original formulation with a zero value channel
+# -----------------------------------------------------------------------------
+
+correct_output_dir_name = get_absolute_path(
+    f"{correct_output_dir_name}/../TiltedV_OverlandFlow_MGSemi"
+)
+correct_root = "TiltedV_OverlandFlow"
+## need to change the channel slopes for Overland Flow which is cell-centered
+overland.TopoSlopesX.Type = "Constant"
+overland.TopoSlopesX.GeomNames = "left right channel"
+overland.TopoSlopesX.Geom.left.Value = -0.01
+overland.TopoSlopesX.Geom.right.Value = 0.01
+overland.TopoSlopesX.Geom.channel.Value = 0.00
+overland.TopoSlopesY.Type = "Constant"
+overland.TopoSlopesY.GeomNames = "domain"
+overland.TopoSlopesY.Geom.domain.Value = 0.01
+overland.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+run_name = "TiltedV_OverlandFlow_JacTrue_MGSemi"
+overland.set_name(run_name)
+overland.Solver.Linear.Preconditioner = "MGSemi"
+print("##########")
+print(f"Running {run_name} Jacobian True MGSemi")
+new_output_dir_name = os.path.join(test_dir_name, f"{run_name}")
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if not check_output(run_name, correct_root, correct_output_dir_name):
+    passed = False
+
+run_name = "TiltedV_OverlandFlow_JacFalse_MGSemi"
+overland.set_name(run_name)
+overland.Solver.Nonlinear.UseJacobian = False
+print("##########")
+print(f"Running {run_name} Jacobian False MGSemi")
+new_output_dir_name = os.path.join(test_dir_name, f"{run_name}")
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if not check_output(run_name, correct_root, correct_output_dir_name):
+    passed = False
+
+
+# -----------------------------------------------------------------------------
+# Diffusive wave (DWE) formulation without the zero channel
+# Note: The difference in configuration here is to be consistent with the way
+# the upwinding is handled for the new and original formulations.
+# Tests with Jacobian True and False and run for both MGSemi preconditioner
+# Results should be almost identical for the different configurations
+# -----------------------------------------------------------------------------
+overland.TopoSlopesX.Type = "Constant"
+overland.TopoSlopesX.GeomNames = "left right channel"
+overland.TopoSlopesX.Geom.left.Value = -0.01
+overland.TopoSlopesX.Geom.right.Value = 0.01
+overland.TopoSlopesX.Geom.channel.Value = 0.01
+
+overland.TopoSlopesY.Type = "Constant"
+overland.TopoSlopesY.GeomNames = "domain"
+overland.TopoSlopesY.Geom.domain.Value = 0.01
+
+# run with DWE Finite Difference Jacobian
+overland.Patch.z_upper.BCPressure.Type = "OverlandDiffusive"
+overland.Solver.Nonlinear.UseJacobian = False
+
+correct_output_dir_name = get_absolute_path(
+    f"{correct_output_dir_name}/../TiltedV_OverlandDiff_MGSemi"
+)
+correct_root = "TiltedV_OverlandDiff"
+
+run_name = "TiltedV_OverlandDiff_JacFalse_MGSemi"
+overland.set_name(run_name)
+print("##########")
+print(f"Running {run_name}")
+new_output_dir_name = os.path.join(test_dir_name, f"{run_name}")
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if not check_output(run_name, correct_root, correct_output_dir_name):
+    passed = False
+
+# run with DWE and analytical jacobian
+run_name = "TiltedV_OverlandDiff_JacTrue_MGSemi"
+overland.set_name(run_name)
+overland.Solver.Nonlinear.UseJacobian = True
+overland.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
+
+print("##########")
+print(f"Running {run_name} Jacobian True")
+new_output_dir_name = os.path.join(test_dir_name, f"{run_name}")
+mkdir(new_output_dir_name)
+overland.run(working_directory=new_output_dir_name)
+if not check_output(run_name, correct_root, correct_output_dir_name):
+    passed = False
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/pfmg.py b/test/python/pfmg.py
index 32f214742..22ce8ffd1 100644
--- a/test/python/pfmg.py
+++ b/test/python/pfmg.py
@@ -1,15 +1,18 @@
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 #  This runs the basic pfmg test case based off of default richards
 #  This run, as written in this input file, should take
 #  3 nonlinear iterations.
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
-pfmg = Run("pfmg", __file__)
+run_name = "pfmg"
+pfmg = Run(run_name, __file__)
 
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 
 pfmg.FileVersion = 4
 
@@ -17,9 +20,9 @@
 pfmg.Process.Topology.Q = 1
 pfmg.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 pfmg.ComputationalGrid.Lower.X = -10.0
 pfmg.ComputationalGrid.Lower.Y = 10.0
@@ -33,22 +36,24 @@
 pfmg.ComputationalGrid.NY = 10
 pfmg.ComputationalGrid.NZ = 8
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.GeomInput.Names = 'domain_input background_input source_region_input concen_region_input'
+pfmg.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.GeomInput.domain_input.InputType = 'Box'
-pfmg.GeomInput.domain_input.GeomName = 'domain'
+pfmg.GeomInput.domain_input.InputType = "Box"
+pfmg.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 pfmg.Geom.domain.Lower.X = -10.0
 pfmg.Geom.domain.Lower.Y = 10.0
@@ -58,18 +63,18 @@
 pfmg.Geom.domain.Upper.Y = 170.0
 pfmg.Geom.domain.Upper.Z = 9.0
 
-pfmg.Geom.domain.Patches = 'left right front back bottom top'
+pfmg.Geom.domain.Patches = "left right front back bottom top"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.GeomInput.background_input.InputType = 'Box'
-pfmg.GeomInput.background_input.GeomName = 'background'
+pfmg.GeomInput.background_input.InputType = "Box"
+pfmg.GeomInput.background_input.GeomName = "background"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 pfmg.Geom.background.Lower.X = -99999999.0
 pfmg.Geom.background.Lower.Y = -99999999.0
@@ -79,16 +84,16 @@
 pfmg.Geom.background.Upper.Y = 99999999.0
 pfmg.Geom.background.Upper.Z = 99999999.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.GeomInput.source_region_input.InputType = 'Box'
-pfmg.GeomInput.source_region_input.GeomName = 'source_region'
+pfmg.GeomInput.source_region_input.InputType = "Box"
+pfmg.GeomInput.source_region_input.GeomName = "source_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 pfmg.Geom.source_region.Lower.X = 65.56
 pfmg.Geom.source_region.Lower.Y = 79.34
@@ -98,16 +103,16 @@
 pfmg.Geom.source_region.Upper.Y = 89.99
 pfmg.Geom.source_region.Upper.Z = 5.5
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.GeomInput.concen_region_input.InputType = 'Box'
-pfmg.GeomInput.concen_region_input.GeomName = 'concen_region'
+pfmg.GeomInput.concen_region_input.InputType = "Box"
+pfmg.GeomInput.concen_region_input.GeomName = "concen_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 pfmg.Geom.concen_region.Lower.X = 60.0
 pfmg.Geom.concen_region.Lower.Y = 80.0
@@ -117,228 +122,264 @@
 pfmg.Geom.concen_region.Upper.Y = 100.0
 pfmg.Geom.concen_region.Upper.Z = 6.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Geom.Perm.Names = 'background'
+pfmg.Geom.Perm.Names = "background"
 
-pfmg.Geom.background.Perm.Type = 'Constant'
+pfmg.Geom.background.Perm.Type = "Constant"
 pfmg.Geom.background.Perm.Value = 4.0
 
-pfmg.Perm.TensorType = 'TensorByGeom'
+pfmg.Perm.TensorType = "TensorByGeom"
 
-pfmg.Geom.Perm.TensorByGeom.Names = 'background'
+pfmg.Geom.Perm.TensorByGeom.Names = "background"
 
 pfmg.Geom.background.Perm.TensorValX = 1.0
 pfmg.Geom.background.Perm.TensorValY = 1.0
 pfmg.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.SpecificStorage.Type = 'Constant'
-pfmg.SpecificStorage.GeomNames = 'domain'
+pfmg.SpecificStorage.Type = "Constant"
+pfmg.SpecificStorage.GeomNames = "domain"
 pfmg.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Phase.Names = 'water'
+pfmg.Phase.Names = "water"
 
-pfmg.Phase.water.Density.Type = 'Constant'
+pfmg.Phase.water.Density.Type = "Constant"
 pfmg.Phase.water.Density.Value = 1.0
 
-pfmg.Phase.water.Viscosity.Type = 'Constant'
+pfmg.Phase.water.Viscosity.Type = "Constant"
 pfmg.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Contaminants.Names = ''
+pfmg.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Geom.Retardation.GeomNames = ''
+pfmg.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 pfmg.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 pfmg.TimingInfo.BaseUnit = 1.0
 pfmg.TimingInfo.StartCount = 0
 pfmg.TimingInfo.StartTime = 0.0
 pfmg.TimingInfo.StopTime = 0.010
 pfmg.TimingInfo.DumpInterval = -1
-pfmg.TimeStep.Type = 'Constant'
+pfmg.TimeStep.Type = "Constant"
 pfmg.TimeStep.Value = 0.001
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Geom.Porosity.GeomNames = 'background'
-pfmg.Geom.background.Porosity.Type = 'Constant'
+pfmg.Geom.Porosity.GeomNames = "background"
+pfmg.Geom.background.Porosity.Type = "Constant"
 pfmg.Geom.background.Porosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Domain.GeomName = 'domain'
+pfmg.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Phase.RelPerm.Type = 'VanGenuchten'
-pfmg.Phase.RelPerm.GeomNames = 'domain'
+pfmg.Phase.RelPerm.Type = "VanGenuchten"
+pfmg.Phase.RelPerm.GeomNames = "domain"
 pfmg.Geom.domain.RelPerm.Alpha = 0.005
 pfmg.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.Phase.Saturation.Type = 'VanGenuchten'
-pfmg.Phase.Saturation.GeomNames = 'domain'
+pfmg.Phase.Saturation.Type = "VanGenuchten"
+pfmg.Phase.Saturation.GeomNames = "domain"
 pfmg.Geom.domain.Saturation.Alpha = 0.005
 pfmg.Geom.domain.Saturation.N = 2.0
 pfmg.Geom.domain.Saturation.SRes = 0.2
 pfmg.Geom.domain.Saturation.SSat = 0.99
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Wells.Names = ''
+pfmg.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Cycle.Names = 'constant'
-pfmg.Cycle.constant.Names = 'alltime'
+pfmg.Cycle.Names = "constant"
+pfmg.Cycle.constant.Names = "alltime"
 pfmg.Cycle.constant.alltime.Length = 1
 pfmg.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.BCPressure.PatchNames = 'left right front back bottom top'
+pfmg.BCPressure.PatchNames = "left right front back bottom top"
 
-pfmg.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-pfmg.Patch.left.BCPressure.Cycle = 'constant'
-pfmg.Patch.left.BCPressure.RefGeom = 'domain'
-pfmg.Patch.left.BCPressure.RefPatch = 'bottom'
+pfmg.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+pfmg.Patch.left.BCPressure.Cycle = "constant"
+pfmg.Patch.left.BCPressure.RefGeom = "domain"
+pfmg.Patch.left.BCPressure.RefPatch = "bottom"
 pfmg.Patch.left.BCPressure.alltime.Value = 5.0
 
-pfmg.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-pfmg.Patch.right.BCPressure.Cycle = 'constant'
-pfmg.Patch.right.BCPressure.RefGeom = 'domain'
-pfmg.Patch.right.BCPressure.RefPatch = 'bottom'
+pfmg.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+pfmg.Patch.right.BCPressure.Cycle = "constant"
+pfmg.Patch.right.BCPressure.RefGeom = "domain"
+pfmg.Patch.right.BCPressure.RefPatch = "bottom"
 pfmg.Patch.right.BCPressure.alltime.Value = 3.0
 
-pfmg.Patch.front.BCPressure.Type = 'FluxConst'
-pfmg.Patch.front.BCPressure.Cycle = 'constant'
+pfmg.Patch.front.BCPressure.Type = "FluxConst"
+pfmg.Patch.front.BCPressure.Cycle = "constant"
 pfmg.Patch.front.BCPressure.alltime.Value = 0.0
 
-pfmg.Patch.back.BCPressure.Type = 'FluxConst'
-pfmg.Patch.back.BCPressure.Cycle = 'constant'
+pfmg.Patch.back.BCPressure.Type = "FluxConst"
+pfmg.Patch.back.BCPressure.Cycle = "constant"
 pfmg.Patch.back.BCPressure.alltime.Value = 0.0
 
-pfmg.Patch.bottom.BCPressure.Type = 'FluxConst'
-pfmg.Patch.bottom.BCPressure.Cycle = 'constant'
+pfmg.Patch.bottom.BCPressure.Type = "FluxConst"
+pfmg.Patch.bottom.BCPressure.Cycle = "constant"
 pfmg.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-pfmg.Patch.top.BCPressure.Type = 'FluxConst'
-pfmg.Patch.top.BCPressure.Cycle = 'constant'
+pfmg.Patch.top.BCPressure.Type = "FluxConst"
+pfmg.Patch.top.BCPressure.Cycle = "constant"
 pfmg.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.TopoSlopesX.Type = 'Constant'
-pfmg.TopoSlopesX.GeomNames = 'domain'
+pfmg.TopoSlopesX.Type = "Constant"
+pfmg.TopoSlopesX.GeomNames = "domain"
 pfmg.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.TopoSlopesY.Type = 'Constant'
-pfmg.TopoSlopesY.GeomNames = 'domain'
+pfmg.TopoSlopesY.Type = "Constant"
+pfmg.TopoSlopesY.GeomNames = "domain"
 pfmg.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-pfmg.Mannings.Type = 'Constant'
-pfmg.Mannings.GeomNames = 'domain'
-pfmg.Mannings.Geom.domain.Value = 0.
+pfmg.Mannings.Type = "Constant"
+pfmg.Mannings.GeomNames = "domain"
+pfmg.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.ICPressure.Type = 'HydroStaticPatch'
-pfmg.ICPressure.GeomNames = 'domain'
+pfmg.ICPressure.Type = "HydroStaticPatch"
+pfmg.ICPressure.GeomNames = "domain"
 pfmg.Geom.domain.ICPressure.Value = 3.0
-pfmg.Geom.domain.ICPressure.RefGeom = 'domain'
-pfmg.Geom.domain.ICPressure.RefPatch = 'bottom'
+pfmg.Geom.domain.ICPressure.RefGeom = "domain"
+pfmg.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.PhaseSources.water.Type = 'Constant'
-pfmg.PhaseSources.water.GeomNames = 'background'
+pfmg.PhaseSources.water.Type = "Constant"
+pfmg.PhaseSources.water.GeomNames = "background"
 pfmg.PhaseSources.water.Geom.background.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.KnownSolution = 'NoKnownSolution'
+pfmg.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Solver = 'Richards'
+pfmg.Solver = "Richards"
 pfmg.Solver.MaxIter = 5
 
 pfmg.Solver.Nonlinear.MaxIter = 10
 pfmg.Solver.Nonlinear.ResidualTol = 1e-9
-pfmg.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+pfmg.Solver.Nonlinear.EtaChoice = "EtaConstant"
 pfmg.Solver.Nonlinear.EtaValue = 1e-5
 pfmg.Solver.Nonlinear.UseJacobian = True
 pfmg.Solver.Nonlinear.DerivativeEpsilon = 1e-2
 
 pfmg.Solver.Linear.KrylovDimension = 10
 
-pfmg.Solver.Linear.Preconditioner = 'PFMG'
-pfmg.Solver.Linear.Preconditioner.PFMG.Smoother = 'WJacobi'
+pfmg.Solver.Linear.Preconditioner = "PFMG"
+pfmg.Solver.Linear.Preconditioner.PFMG.Smoother = "WJacobi"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/pfmg')
-mkdir(dir_name)
-pfmg.run(working_directory=dir_name)
-
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/pfmg")
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+pfmg.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+for i in range(6):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/pfmg_galerkin.py b/test/python/pfmg_galerkin.py
index ad3a951b3..99b6f8e1c 100644
--- a/test/python/pfmg_galerkin.py
+++ b/test/python/pfmg_galerkin.py
@@ -1,15 +1,18 @@
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 #  This runs the basic pfmg test case based off of default richards
 #  This run, as written in this input file, should take
 #  3 nonlinear iterations.
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
-pfmg = Run("pfmg_galerkin", __file__)
+run_name = "pfmg_galerkin"
+pfmg = Run(run_name, __file__)
 
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 
 pfmg.FileVersion = 4
 
@@ -17,9 +20,9 @@
 pfmg.Process.Topology.Q = 1
 pfmg.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 pfmg.ComputationalGrid.Lower.X = -10.0
 pfmg.ComputationalGrid.Lower.Y = 10.0
@@ -33,22 +36,24 @@
 pfmg.ComputationalGrid.NY = 10
 pfmg.ComputationalGrid.NZ = 8
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.GeomInput.Names = 'domain_input background_input source_region_input concen_region_input'
+pfmg.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.GeomInput.domain_input.InputType = 'Box'
-pfmg.GeomInput.domain_input.GeomName = 'domain'
+pfmg.GeomInput.domain_input.InputType = "Box"
+pfmg.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 pfmg.Geom.domain.Lower.X = -10.0
 pfmg.Geom.domain.Lower.Y = 10.0
@@ -58,18 +63,18 @@
 pfmg.Geom.domain.Upper.Y = 170.0
 pfmg.Geom.domain.Upper.Z = 9.0
 
-pfmg.Geom.domain.Patches = 'left right front back bottom top'
+pfmg.Geom.domain.Patches = "left right front back bottom top"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.GeomInput.background_input.InputType = 'Box'
-pfmg.GeomInput.background_input.GeomName = 'background'
+pfmg.GeomInput.background_input.InputType = "Box"
+pfmg.GeomInput.background_input.GeomName = "background"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 pfmg.Geom.background.Lower.X = -99999999.0
 pfmg.Geom.background.Lower.Y = -99999999.0
@@ -79,16 +84,16 @@
 pfmg.Geom.background.Upper.Y = 99999999.0
 pfmg.Geom.background.Upper.Z = 99999999.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.GeomInput.source_region_input.InputType = 'Box'
-pfmg.GeomInput.source_region_input.GeomName = 'source_region'
+pfmg.GeomInput.source_region_input.InputType = "Box"
+pfmg.GeomInput.source_region_input.GeomName = "source_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 pfmg.Geom.source_region.Lower.X = 65.56
 pfmg.Geom.source_region.Lower.Y = 79.34
@@ -98,16 +103,16 @@
 pfmg.Geom.source_region.Upper.Y = 89.99
 pfmg.Geom.source_region.Upper.Z = 5.5
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.GeomInput.concen_region_input.InputType = 'Box'
-pfmg.GeomInput.concen_region_input.GeomName = 'concen_region'
+pfmg.GeomInput.concen_region_input.InputType = "Box"
+pfmg.GeomInput.concen_region_input.GeomName = "concen_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 pfmg.Geom.concen_region.Lower.X = 60.0
 pfmg.Geom.concen_region.Lower.Y = 80.0
@@ -117,229 +122,265 @@
 pfmg.Geom.concen_region.Upper.Y = 100.0
 pfmg.Geom.concen_region.Upper.Z = 6.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Geom.Perm.Names = 'background'
+pfmg.Geom.Perm.Names = "background"
 
-pfmg.Geom.background.Perm.Type = 'Constant'
+pfmg.Geom.background.Perm.Type = "Constant"
 pfmg.Geom.background.Perm.Value = 4.0
 
-pfmg.Perm.TensorType = 'TensorByGeom'
+pfmg.Perm.TensorType = "TensorByGeom"
 
-pfmg.Geom.Perm.TensorByGeom.Names = 'background'
+pfmg.Geom.Perm.TensorByGeom.Names = "background"
 
 pfmg.Geom.background.Perm.TensorValX = 1.0
 pfmg.Geom.background.Perm.TensorValY = 1.0
 pfmg.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.SpecificStorage.Type = 'Constant'
-pfmg.SpecificStorage.GeomNames = 'domain'
+pfmg.SpecificStorage.Type = "Constant"
+pfmg.SpecificStorage.GeomNames = "domain"
 pfmg.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Phase.Names = 'water'
+pfmg.Phase.Names = "water"
 
-pfmg.Phase.water.Density.Type = 'Constant'
+pfmg.Phase.water.Density.Type = "Constant"
 pfmg.Phase.water.Density.Value = 1.0
 
-pfmg.Phase.water.Viscosity.Type = 'Constant'
+pfmg.Phase.water.Viscosity.Type = "Constant"
 pfmg.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Contaminants.Names = ''
+pfmg.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Geom.Retardation.GeomNames = ''
+pfmg.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 pfmg.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 pfmg.TimingInfo.BaseUnit = 1.0
 pfmg.TimingInfo.StartCount = 0
 pfmg.TimingInfo.StartTime = 0.0
 pfmg.TimingInfo.StopTime = 0.010
 pfmg.TimingInfo.DumpInterval = -1
-pfmg.TimeStep.Type = 'Constant'
+pfmg.TimeStep.Type = "Constant"
 pfmg.TimeStep.Value = 0.001
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Geom.Porosity.GeomNames = 'background'
-pfmg.Geom.background.Porosity.Type = 'Constant'
+pfmg.Geom.Porosity.GeomNames = "background"
+pfmg.Geom.background.Porosity.Type = "Constant"
 pfmg.Geom.background.Porosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Domain.GeomName = 'domain'
+pfmg.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Phase.RelPerm.Type = 'VanGenuchten'
-pfmg.Phase.RelPerm.GeomNames = 'domain'
+pfmg.Phase.RelPerm.Type = "VanGenuchten"
+pfmg.Phase.RelPerm.GeomNames = "domain"
 pfmg.Geom.domain.RelPerm.Alpha = 0.005
 pfmg.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.Phase.Saturation.Type = 'VanGenuchten'
-pfmg.Phase.Saturation.GeomNames = 'domain'
+pfmg.Phase.Saturation.Type = "VanGenuchten"
+pfmg.Phase.Saturation.GeomNames = "domain"
 pfmg.Geom.domain.Saturation.Alpha = 0.005
 pfmg.Geom.domain.Saturation.N = 2.0
 pfmg.Geom.domain.Saturation.SRes = 0.2
 pfmg.Geom.domain.Saturation.SSat = 0.99
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Wells.Names = ''
+pfmg.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Cycle.Names = 'constant'
-pfmg.Cycle.constant.Names = 'alltime'
+pfmg.Cycle.Names = "constant"
+pfmg.Cycle.constant.Names = "alltime"
 pfmg.Cycle.constant.alltime.Length = 1
 pfmg.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.BCPressure.PatchNames = 'left right front back bottom top'
+pfmg.BCPressure.PatchNames = "left right front back bottom top"
 
-pfmg.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-pfmg.Patch.left.BCPressure.Cycle = 'constant'
-pfmg.Patch.left.BCPressure.RefGeom = 'domain'
-pfmg.Patch.left.BCPressure.RefPatch = 'bottom'
+pfmg.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+pfmg.Patch.left.BCPressure.Cycle = "constant"
+pfmg.Patch.left.BCPressure.RefGeom = "domain"
+pfmg.Patch.left.BCPressure.RefPatch = "bottom"
 pfmg.Patch.left.BCPressure.alltime.Value = 5.0
 
-pfmg.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-pfmg.Patch.right.BCPressure.Cycle = 'constant'
-pfmg.Patch.right.BCPressure.RefGeom = 'domain'
-pfmg.Patch.right.BCPressure.RefPatch = 'bottom'
+pfmg.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+pfmg.Patch.right.BCPressure.Cycle = "constant"
+pfmg.Patch.right.BCPressure.RefGeom = "domain"
+pfmg.Patch.right.BCPressure.RefPatch = "bottom"
 pfmg.Patch.right.BCPressure.alltime.Value = 3.0
 
-pfmg.Patch.front.BCPressure.Type = 'FluxConst'
-pfmg.Patch.front.BCPressure.Cycle = 'constant'
+pfmg.Patch.front.BCPressure.Type = "FluxConst"
+pfmg.Patch.front.BCPressure.Cycle = "constant"
 pfmg.Patch.front.BCPressure.alltime.Value = 0.0
 
-pfmg.Patch.back.BCPressure.Type = 'FluxConst'
-pfmg.Patch.back.BCPressure.Cycle = 'constant'
+pfmg.Patch.back.BCPressure.Type = "FluxConst"
+pfmg.Patch.back.BCPressure.Cycle = "constant"
 pfmg.Patch.back.BCPressure.alltime.Value = 0.0
 
-pfmg.Patch.bottom.BCPressure.Type = 'FluxConst'
-pfmg.Patch.bottom.BCPressure.Cycle = 'constant'
+pfmg.Patch.bottom.BCPressure.Type = "FluxConst"
+pfmg.Patch.bottom.BCPressure.Cycle = "constant"
 pfmg.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-pfmg.Patch.top.BCPressure.Type = 'FluxConst'
-pfmg.Patch.top.BCPressure.Cycle = 'constant'
+pfmg.Patch.top.BCPressure.Type = "FluxConst"
+pfmg.Patch.top.BCPressure.Cycle = "constant"
 pfmg.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.TopoSlopesX.Type = 'Constant'
-pfmg.TopoSlopesX.GeomNames = 'domain'
+pfmg.TopoSlopesX.Type = "Constant"
+pfmg.TopoSlopesX.GeomNames = "domain"
 pfmg.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.TopoSlopesY.Type = 'Constant'
-pfmg.TopoSlopesY.GeomNames = 'domain'
+pfmg.TopoSlopesY.Type = "Constant"
+pfmg.TopoSlopesY.GeomNames = "domain"
 pfmg.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-pfmg.Mannings.Type = 'Constant'
-pfmg.Mannings.GeomNames = 'domain'
-pfmg.Mannings.Geom.domain.Value = 0.
+pfmg.Mannings.Type = "Constant"
+pfmg.Mannings.GeomNames = "domain"
+pfmg.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.ICPressure.Type = 'HydroStaticPatch'
-pfmg.ICPressure.GeomNames = 'domain'
+pfmg.ICPressure.Type = "HydroStaticPatch"
+pfmg.ICPressure.GeomNames = "domain"
 pfmg.Geom.domain.ICPressure.Value = 3.0
-pfmg.Geom.domain.ICPressure.RefGeom = 'domain'
-pfmg.Geom.domain.ICPressure.RefPatch = 'bottom'
+pfmg.Geom.domain.ICPressure.RefGeom = "domain"
+pfmg.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.PhaseSources.water.Type = 'Constant'
-pfmg.PhaseSources.water.GeomNames = 'background'
+pfmg.PhaseSources.water.Type = "Constant"
+pfmg.PhaseSources.water.GeomNames = "background"
 pfmg.PhaseSources.water.Geom.background.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.KnownSolution = 'NoKnownSolution'
+pfmg.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Solver = 'Richards'
+pfmg.Solver = "Richards"
 pfmg.Solver.MaxIter = 5
 
 pfmg.Solver.Nonlinear.MaxIter = 10
 pfmg.Solver.Nonlinear.ResidualTol = 1e-9
-pfmg.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+pfmg.Solver.Nonlinear.EtaChoice = "EtaConstant"
 pfmg.Solver.Nonlinear.EtaValue = 1e-5
 pfmg.Solver.Nonlinear.UseJacobian = True
 pfmg.Solver.Nonlinear.DerivativeEpsilon = 1e-2
 
 pfmg.Solver.Linear.KrylovDimension = 10
 
-pfmg.Solver.Linear.Preconditioner = 'PFMG'
-pfmg.Solver.Linear.Preconditioner.PFMG.Smoother = 'WJacobi'
-pfmg.Solver.Linear.Preconditioner.PFMG.RAPType = 'Galerkin'
+pfmg.Solver.Linear.Preconditioner = "PFMG"
+pfmg.Solver.Linear.Preconditioner.PFMG.Smoother = "WJacobi"
+pfmg.Solver.Linear.Preconditioner.PFMG.RAPType = "Galerkin"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/pfmg_gal')
-mkdir(dir_name)
-pfmg.run(working_directory=dir_name)
-
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/pfmg_galerkin")
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+pfmg.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+for i in range(6):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/pfmg_octree.py b/test/python/pfmg_octree.py
index 2a9735dcf..d8d1e62b1 100644
--- a/test/python/pfmg_octree.py
+++ b/test/python/pfmg_octree.py
@@ -1,15 +1,18 @@
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 #  This runs the basic pfmg test case based off of default richards
 #  This run, as written in this input file, should take
 #  3 nonlinear iterations.
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
-pfmg = Run("pfmg_octree", __file__)
+run_name = "pfmg"
+pfmg = Run(run_name, __file__)
 
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 
 pfmg.FileVersion = 4
 
@@ -17,9 +20,9 @@
 pfmg.Process.Topology.Q = 1
 pfmg.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 pfmg.ComputationalGrid.Lower.X = -10.0
 pfmg.ComputationalGrid.Lower.Y = 10.0
@@ -33,22 +36,24 @@
 pfmg.ComputationalGrid.NY = 10
 pfmg.ComputationalGrid.NZ = 8
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.GeomInput.Names = 'domain_input background_input source_region_input concen_region_input'
+pfmg.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.GeomInput.domain_input.InputType = 'Box'
-pfmg.GeomInput.domain_input.GeomName = 'domain'
+pfmg.GeomInput.domain_input.InputType = "Box"
+pfmg.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 pfmg.Geom.domain.Lower.X = -10.0
 pfmg.Geom.domain.Lower.Y = 10.0
@@ -58,18 +63,18 @@
 pfmg.Geom.domain.Upper.Y = 170.0
 pfmg.Geom.domain.Upper.Z = 9.0
 
-pfmg.Geom.domain.Patches = 'left right front back bottom top'
+pfmg.Geom.domain.Patches = "left right front back bottom top"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.GeomInput.background_input.InputType = 'Box'
-pfmg.GeomInput.background_input.GeomName = 'background'
+pfmg.GeomInput.background_input.InputType = "Box"
+pfmg.GeomInput.background_input.GeomName = "background"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 pfmg.Geom.background.Lower.X = -99999999.0
 pfmg.Geom.background.Lower.Y = -99999999.0
@@ -79,16 +84,16 @@
 pfmg.Geom.background.Upper.Y = 99999999.0
 pfmg.Geom.background.Upper.Z = 99999999.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.GeomInput.source_region_input.InputType = 'Box'
-pfmg.GeomInput.source_region_input.GeomName = 'source_region'
+pfmg.GeomInput.source_region_input.InputType = "Box"
+pfmg.GeomInput.source_region_input.GeomName = "source_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 pfmg.Geom.source_region.Lower.X = 65.56
 pfmg.Geom.source_region.Lower.Y = 79.34
@@ -98,16 +103,16 @@
 pfmg.Geom.source_region.Upper.Y = 89.99
 pfmg.Geom.source_region.Upper.Z = 5.5
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.GeomInput.concen_region_input.InputType = 'Box'
-pfmg.GeomInput.concen_region_input.GeomName = 'concen_region'
+pfmg.GeomInput.concen_region_input.InputType = "Box"
+pfmg.GeomInput.concen_region_input.GeomName = "concen_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 pfmg.Geom.concen_region.Lower.X = 60.0
 pfmg.Geom.concen_region.Lower.Y = 80.0
@@ -117,229 +122,265 @@
 pfmg.Geom.concen_region.Upper.Y = 100.0
 pfmg.Geom.concen_region.Upper.Z = 6.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Geom.Perm.Names = 'background'
+pfmg.Geom.Perm.Names = "background"
 
-pfmg.Geom.background.Perm.Type = 'Constant'
+pfmg.Geom.background.Perm.Type = "Constant"
 pfmg.Geom.background.Perm.Value = 4.0
 
-pfmg.Perm.TensorType = 'TensorByGeom'
+pfmg.Perm.TensorType = "TensorByGeom"
 
-pfmg.Geom.Perm.TensorByGeom.Names = 'background'
+pfmg.Geom.Perm.TensorByGeom.Names = "background"
 
 pfmg.Geom.background.Perm.TensorValX = 1.0
 pfmg.Geom.background.Perm.TensorValY = 1.0
 pfmg.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.SpecificStorage.Type = 'Constant'
-pfmg.SpecificStorage.GeomNames = 'domain'
+pfmg.SpecificStorage.Type = "Constant"
+pfmg.SpecificStorage.GeomNames = "domain"
 pfmg.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Phase.Names = 'water'
+pfmg.Phase.Names = "water"
 
-pfmg.Phase.water.Density.Type = 'Constant'
+pfmg.Phase.water.Density.Type = "Constant"
 pfmg.Phase.water.Density.Value = 1.0
 
-pfmg.Phase.water.Viscosity.Type = 'Constant'
+pfmg.Phase.water.Viscosity.Type = "Constant"
 pfmg.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Contaminants.Names = ''
+pfmg.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Geom.Retardation.GeomNames = ''
+pfmg.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 pfmg.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 pfmg.TimingInfo.BaseUnit = 1.0
 pfmg.TimingInfo.StartCount = 0
 pfmg.TimingInfo.StartTime = 0.0
 pfmg.TimingInfo.StopTime = 0.010
 pfmg.TimingInfo.DumpInterval = -1
-pfmg.TimeStep.Type = 'Constant'
+pfmg.TimeStep.Type = "Constant"
 pfmg.TimeStep.Value = 0.001
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Geom.Porosity.GeomNames = 'background'
-pfmg.Geom.background.Porosity.Type = 'Constant'
+pfmg.Geom.Porosity.GeomNames = "background"
+pfmg.Geom.background.Porosity.Type = "Constant"
 pfmg.Geom.background.Porosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Domain.GeomName = 'domain'
+pfmg.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Phase.RelPerm.Type = 'VanGenuchten'
-pfmg.Phase.RelPerm.GeomNames = 'domain'
+pfmg.Phase.RelPerm.Type = "VanGenuchten"
+pfmg.Phase.RelPerm.GeomNames = "domain"
 pfmg.Geom.domain.RelPerm.Alpha = 0.005
 pfmg.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.Phase.Saturation.Type = 'VanGenuchten'
-pfmg.Phase.Saturation.GeomNames = 'domain'
+pfmg.Phase.Saturation.Type = "VanGenuchten"
+pfmg.Phase.Saturation.GeomNames = "domain"
 pfmg.Geom.domain.Saturation.Alpha = 0.005
 pfmg.Geom.domain.Saturation.N = 2.0
 pfmg.Geom.domain.Saturation.SRes = 0.2
 pfmg.Geom.domain.Saturation.SSat = 0.99
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Wells.Names = ''
+pfmg.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Cycle.Names = 'constant'
-pfmg.Cycle.constant.Names = 'alltime'
+pfmg.Cycle.Names = "constant"
+pfmg.Cycle.constant.Names = "alltime"
 pfmg.Cycle.constant.alltime.Length = 1
 pfmg.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.BCPressure.PatchNames = 'left right front back bottom top'
+pfmg.BCPressure.PatchNames = "left right front back bottom top"
 
-pfmg.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-pfmg.Patch.left.BCPressure.Cycle = 'constant'
-pfmg.Patch.left.BCPressure.RefGeom = 'domain'
-pfmg.Patch.left.BCPressure.RefPatch = 'bottom'
+pfmg.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+pfmg.Patch.left.BCPressure.Cycle = "constant"
+pfmg.Patch.left.BCPressure.RefGeom = "domain"
+pfmg.Patch.left.BCPressure.RefPatch = "bottom"
 pfmg.Patch.left.BCPressure.alltime.Value = 5.0
 
-pfmg.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-pfmg.Patch.right.BCPressure.Cycle = 'constant'
-pfmg.Patch.right.BCPressure.RefGeom = 'domain'
-pfmg.Patch.right.BCPressure.RefPatch = 'bottom'
+pfmg.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+pfmg.Patch.right.BCPressure.Cycle = "constant"
+pfmg.Patch.right.BCPressure.RefGeom = "domain"
+pfmg.Patch.right.BCPressure.RefPatch = "bottom"
 pfmg.Patch.right.BCPressure.alltime.Value = 3.0
 
-pfmg.Patch.front.BCPressure.Type = 'FluxConst'
-pfmg.Patch.front.BCPressure.Cycle = 'constant'
+pfmg.Patch.front.BCPressure.Type = "FluxConst"
+pfmg.Patch.front.BCPressure.Cycle = "constant"
 pfmg.Patch.front.BCPressure.alltime.Value = 0.0
 
-pfmg.Patch.back.BCPressure.Type = 'FluxConst'
-pfmg.Patch.back.BCPressure.Cycle = 'constant'
+pfmg.Patch.back.BCPressure.Type = "FluxConst"
+pfmg.Patch.back.BCPressure.Cycle = "constant"
 pfmg.Patch.back.BCPressure.alltime.Value = 0.0
 
-pfmg.Patch.bottom.BCPressure.Type = 'FluxConst'
-pfmg.Patch.bottom.BCPressure.Cycle = 'constant'
+pfmg.Patch.bottom.BCPressure.Type = "FluxConst"
+pfmg.Patch.bottom.BCPressure.Cycle = "constant"
 pfmg.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-pfmg.Patch.top.BCPressure.Type = 'FluxConst'
-pfmg.Patch.top.BCPressure.Cycle = 'constant'
+pfmg.Patch.top.BCPressure.Type = "FluxConst"
+pfmg.Patch.top.BCPressure.Cycle = "constant"
 pfmg.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.TopoSlopesX.Type = 'Constant'
-pfmg.TopoSlopesX.GeomNames = 'domain'
+pfmg.TopoSlopesX.Type = "Constant"
+pfmg.TopoSlopesX.GeomNames = "domain"
 pfmg.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.TopoSlopesY.Type = 'Constant'
-pfmg.TopoSlopesY.GeomNames = 'domain'
+pfmg.TopoSlopesY.Type = "Constant"
+pfmg.TopoSlopesY.GeomNames = "domain"
 pfmg.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-pfmg.Mannings.Type = 'Constant'
-pfmg.Mannings.GeomNames = 'domain'
-pfmg.Mannings.Geom.domain.Value = 0.
+pfmg.Mannings.Type = "Constant"
+pfmg.Mannings.GeomNames = "domain"
+pfmg.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-pfmg.ICPressure.Type = 'HydroStaticPatch'
-pfmg.ICPressure.GeomNames = 'domain'
+pfmg.ICPressure.Type = "HydroStaticPatch"
+pfmg.ICPressure.GeomNames = "domain"
 pfmg.Geom.domain.ICPressure.Value = 3.0
-pfmg.Geom.domain.ICPressure.RefGeom = 'domain'
-pfmg.Geom.domain.ICPressure.RefPatch = 'bottom'
+pfmg.Geom.domain.ICPressure.RefGeom = "domain"
+pfmg.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.PhaseSources.water.Type = 'Constant'
-pfmg.PhaseSources.water.GeomNames = 'background'
+pfmg.PhaseSources.water.Type = "Constant"
+pfmg.PhaseSources.water.GeomNames = "background"
 pfmg.PhaseSources.water.Geom.background.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.KnownSolution = 'NoKnownSolution'
+pfmg.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-pfmg.Solver = 'Richards'
+pfmg.Solver = "Richards"
 pfmg.Solver.MaxIter = 5
 
 pfmg.Solver.Nonlinear.MaxIter = 10
 pfmg.Solver.Nonlinear.ResidualTol = 1e-9
-pfmg.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+pfmg.Solver.Nonlinear.EtaChoice = "EtaConstant"
 pfmg.Solver.Nonlinear.EtaValue = 1e-5
 pfmg.Solver.Nonlinear.UseJacobian = True
 pfmg.Solver.Nonlinear.DerivativeEpsilon = 1e-2
 
 pfmg.Solver.Linear.KrylovDimension = 10
 
-pfmg.Solver.Linear.Preconditioner = 'PFMGOctree'
+pfmg.Solver.Linear.Preconditioner = "PFMGOctree"
 
 pfmg.Solver.Linear.Preconditioner.PFMGOctree.BoxSizePowerOf2 = 2
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/pfmg_octree')
-mkdir(dir_name)
-pfmg.run(working_directory=dir_name)
-
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/pfmg_octree")
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+pfmg.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+for i in range(6):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/reservoir_mpi_test.py b/test/python/reservoir_mpi_test.py
new file mode 100644
index 000000000..a67c239a4
--- /dev/null
+++ b/test/python/reservoir_mpi_test.py
@@ -0,0 +1,381 @@
+# ---------------------------------------------------------
+#  Testing overland flow
+# Running a parking lot sloping slab pointed in 8 directions
+# With a suite of overlandflow BC options
+# ---------------------------------------------------------
+
+from parflow import Run
+from parflow.tools.fs import mkdir, get_absolute_path
+import pandas as pd
+import parflow as pf
+from parflow.tools.compare import pf_test_file_with_abs
+import numpy as np
+import sys
+
+run_name = "reservoir_mpi_test"
+sloping_slab = Run(run_name, __file__)
+
+# ---------------------------------------------------------
+
+sloping_slab.FileVersion = 4
+
+sloping_slab.Process.Topology.P = 1
+sloping_slab.Process.Topology.Q = 1
+sloping_slab.Process.Topology.R = 1
+
+# ---------------------------------------------------------
+# Computational Grid
+# ---------------------------------------------------------
+
+sloping_slab.ComputationalGrid.Lower.X = 0.0
+sloping_slab.ComputationalGrid.Lower.Y = 0.0
+sloping_slab.ComputationalGrid.Lower.Z = 0.0
+
+sloping_slab.ComputationalGrid.NX = 5
+sloping_slab.ComputationalGrid.NY = 5
+sloping_slab.ComputationalGrid.NZ = 1
+
+sloping_slab.ComputationalGrid.DX = 10.0
+sloping_slab.ComputationalGrid.DY = 10.0
+sloping_slab.ComputationalGrid.DZ = 0.05
+
+# ---------------------------------------------------------
+# The Names of the GeomInputs
+# ---------------------------------------------------------
+
+sloping_slab.GeomInput.Names = "domaininput"
+sloping_slab.GeomInput.domaininput.GeomName = "domain"
+sloping_slab.GeomInput.domaininput.InputType = "Box"
+
+# ---------------------------------------------------------
+# Domain Geometry
+# ---------------------------------------------------------
+
+sloping_slab.Geom.domain.Lower.X = 0.0
+sloping_slab.Geom.domain.Lower.Y = 0.0
+sloping_slab.Geom.domain.Lower.Z = 0.0
+
+sloping_slab.Geom.domain.Upper.X = 50.0
+sloping_slab.Geom.domain.Upper.Y = 50.0
+sloping_slab.Geom.domain.Upper.Z = 0.05
+sloping_slab.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
+
+# -----------------------------------------------------------------------------
+# Perm
+# -----------------------------------------------------------------------------
+
+sloping_slab.Geom.Perm.Names = "domain"
+sloping_slab.Geom.domain.Perm.Type = "Constant"
+sloping_slab.Geom.domain.Perm.Value = 0.0000001
+
+sloping_slab.Perm.TensorType = "TensorByGeom"
+
+sloping_slab.Geom.Perm.TensorByGeom.Names = "domain"
+
+sloping_slab.Geom.domain.Perm.TensorValX = 1.0
+sloping_slab.Geom.domain.Perm.TensorValY = 1.0
+sloping_slab.Geom.domain.Perm.TensorValZ = 1.0
+
+# -----------------------------------------------------------------------------
+# Specific Storage
+# -----------------------------------------------------------------------------
+
+sloping_slab.SpecificStorage.Type = "Constant"
+sloping_slab.SpecificStorage.GeomNames = "domain"
+sloping_slab.Geom.domain.SpecificStorage.Value = 1.0e-4
+
+# -----------------------------------------------------------------------------
+# Phases
+# -----------------------------------------------------------------------------
+
+sloping_slab.Phase.Names = "water"
+
+sloping_slab.Phase.water.Density.Type = "Constant"
+sloping_slab.Phase.water.Density.Value = 1.0
+
+sloping_slab.Phase.water.Viscosity.Type = "Constant"
+sloping_slab.Phase.water.Viscosity.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Contaminants
+# -----------------------------------------------------------------------------
+
+sloping_slab.Contaminants.Names = ""
+
+# -----------------------------------------------------------------------------
+# Retardation
+# -----------------------------------------------------------------------------
+
+sloping_slab.Geom.Retardation.GeomNames = ""
+
+# -----------------------------------------------------------------------------
+# Gravity
+# -----------------------------------------------------------------------------
+
+sloping_slab.Gravity = 1.0
+
+# -----------------------------------------------------------------------------
+# Setup timing info
+# -----------------------------------------------------------------------------
+
+sloping_slab.TimingInfo.BaseUnit = 0.05
+sloping_slab.TimingInfo.StartCount = 0
+sloping_slab.TimingInfo.StartTime = 0.0
+sloping_slab.TimingInfo.StopTime = 1.0
+sloping_slab.TimingInfo.DumpInterval = -2
+sloping_slab.TimeStep.Type = "Constant"
+sloping_slab.TimeStep.Value = 0.05
+
+# -----------------------------------------------------------------------------
+# Porosity
+# -----------------------------------------------------------------------------
+
+sloping_slab.Geom.Porosity.GeomNames = "domain"
+sloping_slab.Geom.domain.Porosity.Type = "Constant"
+sloping_slab.Geom.domain.Porosity.Value = 0.01
+
+# -----------------------------------------------------------------------------
+# Domain
+# -----------------------------------------------------------------------------
+
+sloping_slab.Domain.GeomName = "domain"
+
+# -----------------------------------------------------------------------------
+# Relative Permeability
+# -----------------------------------------------------------------------------
+
+sloping_slab.Phase.RelPerm.Type = "VanGenuchten"
+sloping_slab.Phase.RelPerm.GeomNames = "domain"
+
+sloping_slab.Geom.domain.RelPerm.Alpha = 6.0
+sloping_slab.Geom.domain.RelPerm.N = 2.0
+
+# ---------------------------------------------------------
+# Saturation
+# ---------------------------------------------------------
+
+sloping_slab.Phase.Saturation.Type = "VanGenuchten"
+sloping_slab.Phase.Saturation.GeomNames = "domain"
+
+sloping_slab.Geom.domain.Saturation.Alpha = 6.0
+sloping_slab.Geom.domain.Saturation.N = 2.0
+sloping_slab.Geom.domain.Saturation.SRes = 0.2
+sloping_slab.Geom.domain.Saturation.SSat = 1.0
+
+# -----------------------------------------------------------------------------
+# Wells
+# -----------------------------------------------------------------------------
+
+sloping_slab.Wells.Names = ""
+
+# -----------------------------------------------------------------------------
+# Time Cycles
+# -----------------------------------------------------------------------------
+
+sloping_slab.Cycle.Names = "constant rainrec"
+sloping_slab.Cycle.constant.Names = "alltime"
+sloping_slab.Cycle.constant.alltime.Length = 1
+sloping_slab.Cycle.constant.Repeat = -1
+
+# rainfall and recession time periods are defined here
+# rain for 1 hour, recession for 2 hours
+
+sloping_slab.Cycle.rainrec.Names = "rain rec"
+sloping_slab.Cycle.rainrec.rain.Length = 2
+sloping_slab.Cycle.rainrec.rec.Length = 300
+sloping_slab.Cycle.rainrec.Repeat = -1
+
+# -----------------------------------------------------------------------------
+# Boundary Conditions: Pressure
+# -----------------------------------------------------------------------------
+
+sloping_slab.BCPressure.PatchNames = sloping_slab.Geom.domain.Patches
+
+sloping_slab.Patch.x_lower.BCPressure.Type = "FluxConst"
+sloping_slab.Patch.x_lower.BCPressure.Cycle = "constant"
+sloping_slab.Patch.x_lower.BCPressure.alltime.Value = 0.0
+
+sloping_slab.Patch.y_lower.BCPressure.Type = "FluxConst"
+sloping_slab.Patch.y_lower.BCPressure.Cycle = "constant"
+sloping_slab.Patch.y_lower.BCPressure.alltime.Value = 0.0
+
+sloping_slab.Patch.z_lower.BCPressure.Type = "FluxConst"
+sloping_slab.Patch.z_lower.BCPressure.Cycle = "constant"
+sloping_slab.Patch.z_lower.BCPressure.alltime.Value = 0.0
+
+sloping_slab.Patch.x_upper.BCPressure.Type = "FluxConst"
+sloping_slab.Patch.x_upper.BCPressure.Cycle = "constant"
+sloping_slab.Patch.x_upper.BCPressure.alltime.Value = 0.0
+
+sloping_slab.Patch.y_upper.BCPressure.Type = "FluxConst"
+sloping_slab.Patch.y_upper.BCPressure.Cycle = "constant"
+sloping_slab.Patch.y_upper.BCPressure.alltime.Value = 0.0
+
+## overland flow boundary condition with very heavy rainfall then slight ET
+sloping_slab.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+sloping_slab.Patch.z_upper.BCPressure.Cycle = "rainrec"
+sloping_slab.Patch.z_upper.BCPressure.rain.Value = -0.01
+sloping_slab.Patch.z_upper.BCPressure.rec.Value = 0.0000
+
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
+
+sloping_slab.Mannings.Type = "Constant"
+sloping_slab.Mannings.GeomNames = "domain"
+sloping_slab.Mannings.Geom.domain.Value = 3.0e-6
+
+# -----------------------------------------------------------------------------
+# Phase sources:
+# -----------------------------------------------------------------------------
+
+sloping_slab.PhaseSources.water.Type = "Constant"
+sloping_slab.PhaseSources.water.GeomNames = "domain"
+sloping_slab.PhaseSources.water.Geom.domain.Value = 0.0
+
+# -----------------------------------------------------------------------------
+# Exact solution specification for error calculations
+# -----------------------------------------------------------------------------
+
+sloping_slab.KnownSolution = "NoKnownSolution"
+
+# -----------------------------------------------------------------------------
+# Set solver parameters
+# -----------------------------------------------------------------------------
+
+sloping_slab.Solver = "Richards"
+sloping_slab.Solver.MaxIter = 2500
+
+sloping_slab.Solver.Nonlinear.MaxIter = 50
+sloping_slab.Solver.Nonlinear.ResidualTol = 1e-9
+sloping_slab.Solver.Nonlinear.EtaChoice = "EtaConstant"
+sloping_slab.Solver.Nonlinear.EtaValue = 0.01
+sloping_slab.Solver.Nonlinear.UseJacobian = False
+
+sloping_slab.Solver.Nonlinear.DerivativeEpsilon = 1e-15
+sloping_slab.Solver.Nonlinear.StepTol = 1e-20
+sloping_slab.Solver.Nonlinear.Globalization = "LineSearch"
+sloping_slab.Solver.Linear.KrylovDimension = 20
+sloping_slab.Solver.Linear.MaxRestart = 2
+
+sloping_slab.Solver.Linear.Preconditioner = "PFMG"
+sloping_slab.Solver.PrintSubsurf = False
+sloping_slab.Solver.Drop = 1e-20
+sloping_slab.Solver.AbsTol = 1e-10
+
+sloping_slab.Solver.OverlandKinematic.Epsilon = 1e-5
+
+sloping_slab.Solver.WriteSiloSubsurfData = False
+sloping_slab.Solver.WriteSiloPressure = False
+sloping_slab.Solver.WriteSiloSlopes = False
+
+sloping_slab.Solver.WriteSiloSaturation = False
+sloping_slab.Solver.WriteSiloConcentration = False
+
+# ---------------------------------------------------------
+# Initial conditions: water pressure
+# ---------------------------------------------------------
+
+# set water table to be at the bottom of the domain, the top layer is initially dry
+sloping_slab.ICPressure.Type = "HydroStaticPatch"
+sloping_slab.ICPressure.GeomNames = "domain"
+sloping_slab.Geom.domain.ICPressure.Value = -3.0
+
+sloping_slab.Geom.domain.ICPressure.RefGeom = "domain"
+sloping_slab.Geom.domain.ICPressure.RefPatch = "z_upper"
+
+
+#   #### Set the slopes. Will flow in -x direction
+sloping_slab.TopoSlopesX.Type = "Constant"
+sloping_slab.TopoSlopesX.GeomNames = "domain"
+sloping_slab.TopoSlopesX.Geom.domain.Value = -0.01
+
+sloping_slab.TopoSlopesY.Type = "Constant"
+sloping_slab.TopoSlopesY.GeomNames = "domain"
+sloping_slab.TopoSlopesY.Geom.domain.Value = 0.0
+
+# turn on analytical jacobian and re-test
+sloping_slab.Solver.Nonlinear.UseJacobian = True
+sloping_slab.Solver.Linear.Preconditioner.PCMatrixType = "PFSymmetric"
+
+
+sloping_slab.Reservoirs.Names = "reservoir"
+sloping_slab.Reservoirs.Overland_Flow_Solver = "OverlandFlow"
+
+sloping_slab.Reservoirs.reservoir.Intake_X = 25
+sloping_slab.Reservoirs.reservoir.Intake_Y = 25
+sloping_slab.Reservoirs.reservoir.Secondary_Intake_X = 25
+sloping_slab.Reservoirs.reservoir.Secondary_Intake_Y = 35
+sloping_slab.Reservoirs.reservoir.Release_X = 35
+sloping_slab.Reservoirs.reservoir.Release_Y = 25
+sloping_slab.Reservoirs.reservoir.Has_Secondary_Intake_Cell = 1
+
+sloping_slab.Reservoirs.reservoir.Max_Storage = 7000000
+sloping_slab.Reservoirs.reservoir.Storage = 5500000.0
+sloping_slab.Reservoirs.reservoir.Min_Release_Storage = 4000000
+sloping_slab.Reservoirs.reservoir.Release_Rate = 1
+
+# run the base case on one processor. We then test that we get the same answer when using different
+# processor configurations
+base_case_dir = get_absolute_path("test_output/reservoir_mpi_test_1_processor")
+mkdir(base_case_dir)
+sloping_slab.run(working_directory=base_case_dir)
+
+# test that when a reservoir spans two ranks we get the correct answer
+sloping_slab.Process.Topology.P = 5
+sloping_slab.Process.Topology.Q = 1
+sloping_slab.Process.Topology.R = 1
+
+passed = True
+
+test1_dir = get_absolute_path("test_output/reservoir_mpi_test_5_processors")
+mkdir(test1_dir)
+sloping_slab.run(working_directory=test1_dir)
+
+
+i = 10
+timestep = str(i).rjust(5, "0")
+
+sig_digits = 10
+abs_value = 1e-18
+
+test_files = ["press"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        test1_dir + filename,
+        base_case_dir + filename,
+        f"Max difference in {filename}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+
+# test that when a reservoir spans three ranks we get the correct answer
+sloping_slab.Process.Topology.P = 5
+sloping_slab.Process.Topology.Q = 5
+sloping_slab.Process.Topology.R = 1
+
+test2_dir = get_absolute_path("test_output/reservoir_mpi_test_25_processors")
+mkdir(test2_dir)
+sloping_slab.run(working_directory=test2_dir)
+
+test_files = ["press"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        test2_dir + filename,
+        base_case_dir + filename,
+        f"Max difference in {filename}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/richards_FBx.py b/test/python/richards_FBx.py
index 325449a5c..a8b976b3e 100644
--- a/test/python/richards_FBx.py
+++ b/test/python/richards_FBx.py
@@ -1,23 +1,26 @@
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 # This runs a test case with the Richards' solver
 # with simple flow domains, like a wall or a fault.
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 
+import sys
 from parflow import Run, write_pfb
 from parflow.tools.fs import get_absolute_path, mkdir, chdir
+from parflow.tools.compare import pf_test_file
 import numpy as np
 
-rich_fbx = Run("richards_FBx", __file__)
+run_name = "richards_FBx"
+rich_fbx = Run(run_name, __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Creating and navigating to output directory
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/rich_fbx')
-mkdir(dir_name)
-chdir(dir_name)
+correct_output_dir_name = get_absolute_path("../correct_output")
+new_output_dir_name = get_absolute_path("test_output/richards_fbx")
+mkdir(new_output_dir_name)
 
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 
 rich_fbx.FileVersion = 4
 
@@ -25,9 +28,9 @@
 rich_fbx.Process.Topology.Q = 1
 rich_fbx.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rich_fbx.ComputationalGrid.Lower.X = 0.0
 rich_fbx.ComputationalGrid.Lower.Y = 0.0
@@ -41,22 +44,22 @@
 rich_fbx.ComputationalGrid.NY = 20
 rich_fbx.ComputationalGrid.NZ = 20
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich_fbx.GeomInput.Names = 'domain_input'
+rich_fbx.GeomInput.Names = "domain_input"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich_fbx.GeomInput.domain_input.InputType = 'Box'
-rich_fbx.GeomInput.domain_input.GeomName = 'domain'
+rich_fbx.GeomInput.domain_input.InputType = "Box"
+rich_fbx.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rich_fbx.Geom.domain.Lower.X = 0.0
 rich_fbx.Geom.domain.Lower.Y = 0.0
@@ -66,231 +69,231 @@
 rich_fbx.Geom.domain.Upper.Y = 20.0
 rich_fbx.Geom.domain.Upper.Z = 20.0
 
-rich_fbx.Geom.domain.Patches = 'left right front back bottom top'
+rich_fbx.Geom.domain.Patches = "left right front back bottom top"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fbx.Geom.Perm.Names = 'domain'
-rich_fbx.Geom.domain.Perm.Type = 'Constant'
+rich_fbx.Geom.Perm.Names = "domain"
+rich_fbx.Geom.domain.Perm.Type = "Constant"
 rich_fbx.Geom.domain.Perm.Value = 1.0
 
-rich_fbx.Perm.TensorType = 'TensorByGeom'
+rich_fbx.Perm.TensorType = "TensorByGeom"
 
-rich_fbx.Geom.Perm.TensorByGeom.Names = 'domain'
+rich_fbx.Geom.Perm.TensorByGeom.Names = "domain"
 
 rich_fbx.Geom.domain.Perm.TensorValX = 1.0
 rich_fbx.Geom.domain.Perm.TensorValY = 1.0
 rich_fbx.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fbx.SpecificStorage.Type = 'Constant'
-rich_fbx.SpecificStorage.GeomNames = 'domain'
+rich_fbx.SpecificStorage.Type = "Constant"
+rich_fbx.SpecificStorage.GeomNames = "domain"
 rich_fbx.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fbx.Phase.Names = 'water'
+rich_fbx.Phase.Names = "water"
 
-rich_fbx.Phase.water.Density.Type = 'Constant'
+rich_fbx.Phase.water.Density.Type = "Constant"
 rich_fbx.Phase.water.Density.Value = 1.0
 
-rich_fbx.Phase.water.Viscosity.Type = 'Constant'
+rich_fbx.Phase.water.Viscosity.Type = "Constant"
 rich_fbx.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fbx.Contaminants.Names = ''
+rich_fbx.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fbx.Geom.Retardation.GeomNames = ''
+rich_fbx.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 rich_fbx.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fbx.TimingInfo.BaseUnit = 10.
+rich_fbx.TimingInfo.BaseUnit = 10.0
 rich_fbx.TimingInfo.StartCount = 0
 rich_fbx.TimingInfo.StartTime = 0.0
 rich_fbx.TimingInfo.StopTime = 100.0
 rich_fbx.TimingInfo.DumpInterval = 10.0
-rich_fbx.TimeStep.Type = 'Constant'
+rich_fbx.TimeStep.Type = "Constant"
 rich_fbx.TimeStep.Value = 10.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fbx.Geom.Porosity.GeomNames = 'domain'
-rich_fbx.Geom.domain.Porosity.Type = 'Constant'
+rich_fbx.Geom.Porosity.GeomNames = "domain"
+rich_fbx.Geom.domain.Porosity.Type = "Constant"
 rich_fbx.Geom.domain.Porosity.Value = 0.25
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fbx.Domain.GeomName = 'domain'
+rich_fbx.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fbx.Phase.RelPerm.Type = 'VanGenuchten'
-rich_fbx.Phase.RelPerm.GeomNames = 'domain'
+rich_fbx.Phase.RelPerm.Type = "VanGenuchten"
+rich_fbx.Phase.RelPerm.GeomNames = "domain"
 rich_fbx.Geom.domain.RelPerm.Alpha = 2.0
 rich_fbx.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich_fbx.Phase.Saturation.Type = 'VanGenuchten'
-rich_fbx.Phase.Saturation.GeomNames = 'domain'
+rich_fbx.Phase.Saturation.Type = "VanGenuchten"
+rich_fbx.Phase.Saturation.GeomNames = "domain"
 rich_fbx.Geom.domain.Saturation.Alpha = 2.0
 rich_fbx.Geom.domain.Saturation.N = 2.0
 rich_fbx.Geom.domain.Saturation.SRes = 0.1
 rich_fbx.Geom.domain.Saturation.SSat = 1.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Flow Barrier in X between cells 10 and 11 in all Z
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rich_fbx.Solver.Nonlinear.FlowBarrierX = True
-rich_fbx.FBx.Type = 'PFBFile'
-rich_fbx.Geom.domain.FBx.FileName = 'Flow_Barrier_X.pfb'
+rich_fbx.FBx.Type = "PFBFile"
+rich_fbx.Geom.domain.FBx.FileName = "Flow_Barrier_X.pfb"
 
 ## write flow boundary file
 FBx_data = np.full((20, 20, 20), 1.0)
 # from cell 10 (index 9) to cell 11
 # reduction of 1E-3
 FBx_data[:, :, 9] = 0.001
-write_pfb(get_absolute_path('Flow_Barrier_X.pfb'), FBx_data)
-rich_fbx.dist('Flow_Barrier_X.pfb')
+write_pfb(get_absolute_path(new_output_dir_name + "/Flow_Barrier_X.pfb"), FBx_data)
+rich_fbx.dist(new_output_dir_name + "/Flow_Barrier_X.pfb")
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fbx.Wells.Names = ''
+rich_fbx.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fbx.Cycle.Names = 'constant'
-rich_fbx.Cycle.constant.Names = 'alltime'
+rich_fbx.Cycle.Names = "constant"
+rich_fbx.Cycle.constant.Names = "alltime"
 rich_fbx.Cycle.constant.alltime.Length = 1
 rich_fbx.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fbx.BCPressure.PatchNames = 'left right front back bottom top'
+rich_fbx.BCPressure.PatchNames = "left right front back bottom top"
 
-rich_fbx.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-rich_fbx.Patch.left.BCPressure.Cycle = 'constant'
-rich_fbx.Patch.left.BCPressure.RefGeom = 'domain'
-rich_fbx.Patch.left.BCPressure.RefPatch = 'bottom'
+rich_fbx.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+rich_fbx.Patch.left.BCPressure.Cycle = "constant"
+rich_fbx.Patch.left.BCPressure.RefGeom = "domain"
+rich_fbx.Patch.left.BCPressure.RefPatch = "bottom"
 rich_fbx.Patch.left.BCPressure.alltime.Value = 11.0
 
-rich_fbx.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-rich_fbx.Patch.right.BCPressure.Cycle = 'constant'
-rich_fbx.Patch.right.BCPressure.RefGeom = 'domain'
-rich_fbx.Patch.right.BCPressure.RefPatch = 'bottom'
+rich_fbx.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+rich_fbx.Patch.right.BCPressure.Cycle = "constant"
+rich_fbx.Patch.right.BCPressure.RefGeom = "domain"
+rich_fbx.Patch.right.BCPressure.RefPatch = "bottom"
 rich_fbx.Patch.right.BCPressure.alltime.Value = 15.0
 
-rich_fbx.Patch.front.BCPressure.Type = 'FluxConst'
-rich_fbx.Patch.front.BCPressure.Cycle = 'constant'
+rich_fbx.Patch.front.BCPressure.Type = "FluxConst"
+rich_fbx.Patch.front.BCPressure.Cycle = "constant"
 rich_fbx.Patch.front.BCPressure.alltime.Value = 0.0
 
-rich_fbx.Patch.back.BCPressure.Type = 'FluxConst'
-rich_fbx.Patch.back.BCPressure.Cycle = 'constant'
+rich_fbx.Patch.back.BCPressure.Type = "FluxConst"
+rich_fbx.Patch.back.BCPressure.Cycle = "constant"
 rich_fbx.Patch.back.BCPressure.alltime.Value = 0.0
 
-rich_fbx.Patch.bottom.BCPressure.Type = 'FluxConst'
-rich_fbx.Patch.bottom.BCPressure.Cycle = 'constant'
+rich_fbx.Patch.bottom.BCPressure.Type = "FluxConst"
+rich_fbx.Patch.bottom.BCPressure.Cycle = "constant"
 rich_fbx.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-rich_fbx.Patch.top.BCPressure.Type = 'FluxConst'
-rich_fbx.Patch.top.BCPressure.Cycle = 'constant'
+rich_fbx.Patch.top.BCPressure.Type = "FluxConst"
+rich_fbx.Patch.top.BCPressure.Cycle = "constant"
 rich_fbx.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich_fbx.TopoSlopesX.Type = 'Constant'
-rich_fbx.TopoSlopesX.GeomNames = 'domain'
+rich_fbx.TopoSlopesX.Type = "Constant"
+rich_fbx.TopoSlopesX.GeomNames = "domain"
 rich_fbx.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich_fbx.TopoSlopesY.Type = 'Constant'
-rich_fbx.TopoSlopesY.GeomNames = 'domain'
+rich_fbx.TopoSlopesY.Type = "Constant"
+rich_fbx.TopoSlopesY.GeomNames = "domain"
 rich_fbx.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich_fbx.Mannings.Type = 'Constant'
-rich_fbx.Mannings.GeomNames = 'domain'
-rich_fbx.Mannings.Geom.domain.Value = 0.
+rich_fbx.Mannings.Type = "Constant"
+rich_fbx.Mannings.GeomNames = "domain"
+rich_fbx.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich_fbx.ICPressure.Type = 'HydroStaticPatch'
-rich_fbx.ICPressure.GeomNames = 'domain'
+rich_fbx.ICPressure.Type = "HydroStaticPatch"
+rich_fbx.ICPressure.GeomNames = "domain"
 rich_fbx.Geom.domain.ICPressure.Value = 13.0
-rich_fbx.Geom.domain.ICPressure.RefGeom = 'domain'
-rich_fbx.Geom.domain.ICPressure.RefPatch = 'bottom'
+rich_fbx.Geom.domain.ICPressure.RefGeom = "domain"
+rich_fbx.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fbx.PhaseSources.water.Type = 'Constant'
-rich_fbx.PhaseSources.water.GeomNames = 'domain'
+rich_fbx.PhaseSources.water.Type = "Constant"
+rich_fbx.PhaseSources.water.GeomNames = "domain"
 rich_fbx.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fbx.KnownSolution = 'NoKnownSolution'
+rich_fbx.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fbx.Solver = 'Richards'
+rich_fbx.Solver = "Richards"
 rich_fbx.Solver.MaxIter = 50000
 
 rich_fbx.Solver.Nonlinear.MaxIter = 100
 rich_fbx.Solver.Nonlinear.ResidualTol = 1e-6
-rich_fbx.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+rich_fbx.Solver.Nonlinear.EtaChoice = "EtaConstant"
 rich_fbx.Solver.Nonlinear.EtaValue = 1e-2
 rich_fbx.Solver.Nonlinear.UseJacobian = True
 
@@ -298,10 +301,33 @@
 
 rich_fbx.Solver.Linear.KrylovDimension = 100
 
-rich_fbx.Solver.Linear.Preconditioner = 'PFMG'
+rich_fbx.Solver.Linear.Preconditioner = "PFMG"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-rich_fbx.run()
+# -----------------------------------------------------------------------------
+
+rich_fbx.run(working_directory=new_output_dir_name)
+passed = True
+for i in range(11):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/richards_FBy.py b/test/python/richards_FBy.py
index 924d2ca45..bdcb751c7 100644
--- a/test/python/richards_FBy.py
+++ b/test/python/richards_FBy.py
@@ -1,23 +1,26 @@
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 # This runs a test case with the Richards' solver
 # with simple flow domains, like a wall or a fault.
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 
+import sys
 from parflow import Run, write_pfb
 from parflow.tools.fs import get_absolute_path, mkdir, chdir
+from parflow.tools.compare import pf_test_file
 import numpy as np
 
-rich_fby = Run("richards_FBy", __file__)
+run_name = "richards_FBy"
+rich_fby = Run(run_name, __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Creating and navigating to output directory
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/rich_fby')
-mkdir(dir_name)
-chdir(dir_name)
+correct_output_dir_name = get_absolute_path("../correct_output")
+new_output_dir_name = get_absolute_path("test_output/richards_fby")
+mkdir(new_output_dir_name)
 
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 
 rich_fby.FileVersion = 4
 
@@ -25,9 +28,9 @@
 rich_fby.Process.Topology.Q = 1
 rich_fby.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rich_fby.ComputationalGrid.Lower.X = 0.0
 rich_fby.ComputationalGrid.Lower.Y = 0.0
@@ -41,22 +44,22 @@
 rich_fby.ComputationalGrid.NY = 20
 rich_fby.ComputationalGrid.NZ = 20
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich_fby.GeomInput.Names = 'domain_input'
+rich_fby.GeomInput.Names = "domain_input"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich_fby.GeomInput.domain_input.InputType = 'Box'
-rich_fby.GeomInput.domain_input.GeomName = 'domain'
+rich_fby.GeomInput.domain_input.InputType = "Box"
+rich_fby.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rich_fby.Geom.domain.Lower.X = 0.0
 rich_fby.Geom.domain.Lower.Y = 0.0
@@ -66,231 +69,231 @@
 rich_fby.Geom.domain.Upper.Y = 20.0
 rich_fby.Geom.domain.Upper.Z = 20.0
 
-rich_fby.Geom.domain.Patches = 'left right front back bottom top'
+rich_fby.Geom.domain.Patches = "left right front back bottom top"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fby.Geom.Perm.Names = 'domain'
-rich_fby.Geom.domain.Perm.Type = 'Constant'
+rich_fby.Geom.Perm.Names = "domain"
+rich_fby.Geom.domain.Perm.Type = "Constant"
 rich_fby.Geom.domain.Perm.Value = 1.0
 
-rich_fby.Perm.TensorType = 'TensorByGeom'
+rich_fby.Perm.TensorType = "TensorByGeom"
 
-rich_fby.Geom.Perm.TensorByGeom.Names = 'domain'
+rich_fby.Geom.Perm.TensorByGeom.Names = "domain"
 
 rich_fby.Geom.domain.Perm.TensorValX = 1.0
 rich_fby.Geom.domain.Perm.TensorValY = 1.0
 rich_fby.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fby.SpecificStorage.Type = 'Constant'
-rich_fby.SpecificStorage.GeomNames = 'domain'
+rich_fby.SpecificStorage.Type = "Constant"
+rich_fby.SpecificStorage.GeomNames = "domain"
 rich_fby.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fby.Phase.Names = 'water'
+rich_fby.Phase.Names = "water"
 
-rich_fby.Phase.water.Density.Type = 'Constant'
+rich_fby.Phase.water.Density.Type = "Constant"
 rich_fby.Phase.water.Density.Value = 1.0
 
-rich_fby.Phase.water.Viscosity.Type = 'Constant'
+rich_fby.Phase.water.Viscosity.Type = "Constant"
 rich_fby.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fby.Contaminants.Names = ''
+rich_fby.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fby.Geom.Retardation.GeomNames = ''
+rich_fby.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 rich_fby.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fby.TimingInfo.BaseUnit = 10.
+rich_fby.TimingInfo.BaseUnit = 10.0
 rich_fby.TimingInfo.StartCount = 0
 rich_fby.TimingInfo.StartTime = 0.0
 rich_fby.TimingInfo.StopTime = 100.0
 rich_fby.TimingInfo.DumpInterval = 10.0
-rich_fby.TimeStep.Type = 'Constant'
+rich_fby.TimeStep.Type = "Constant"
 rich_fby.TimeStep.Value = 10.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fby.Geom.Porosity.GeomNames = 'domain'
-rich_fby.Geom.domain.Porosity.Type = 'Constant'
+rich_fby.Geom.Porosity.GeomNames = "domain"
+rich_fby.Geom.domain.Porosity.Type = "Constant"
 rich_fby.Geom.domain.Porosity.Value = 0.25
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fby.Domain.GeomName = 'domain'
+rich_fby.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fby.Phase.RelPerm.Type = 'VanGenuchten'
-rich_fby.Phase.RelPerm.GeomNames = 'domain'
+rich_fby.Phase.RelPerm.Type = "VanGenuchten"
+rich_fby.Phase.RelPerm.GeomNames = "domain"
 rich_fby.Geom.domain.RelPerm.Alpha = 2.0
 rich_fby.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich_fby.Phase.Saturation.Type = 'VanGenuchten'
-rich_fby.Phase.Saturation.GeomNames = 'domain'
+rich_fby.Phase.Saturation.Type = "VanGenuchten"
+rich_fby.Phase.Saturation.GeomNames = "domain"
 rich_fby.Geom.domain.Saturation.Alpha = 2.0
 rich_fby.Geom.domain.Saturation.N = 2.0
 rich_fby.Geom.domain.Saturation.SRes = 0.1
 rich_fby.Geom.domain.Saturation.SSat = 1.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Flow Barrier in Y between cells 10 and 11 in all Z
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rich_fby.Solver.Nonlinear.FlowBarrierY = True
-rich_fby.FBy.Type = 'PFBFile'
-rich_fby.Geom.domain.FBy.FileName = 'Flow_Barrier_Y.pfb'
+rich_fby.FBy.Type = "PFBFile"
+rich_fby.Geom.domain.FBy.FileName = "Flow_Barrier_Y.pfb"
 
 ## write flow barrier file
 FBy_data = np.full((20, 20, 20), 1.0)
 # from cell 10 (index 9) to cell 11
 # reduction of 1E-3
 FBy_data[:, 9, :] = 0.001
-write_pfb(get_absolute_path('Flow_Barrier_Y.pfb'), FBy_data)
-rich_fby.dist('Flow_Barrier_Y.pfb')
+write_pfb(get_absolute_path(new_output_dir_name + "/Flow_Barrier_Y.pfb"), FBy_data)
+rich_fby.dist(new_output_dir_name + "/Flow_Barrier_Y.pfb")
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fby.Wells.Names = ''
+rich_fby.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fby.Cycle.Names = 'constant'
-rich_fby.Cycle.constant.Names = 'alltime'
+rich_fby.Cycle.Names = "constant"
+rich_fby.Cycle.constant.Names = "alltime"
 rich_fby.Cycle.constant.alltime.Length = 1
 rich_fby.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fby.BCPressure.PatchNames = 'left right front back bottom top'
+rich_fby.BCPressure.PatchNames = "left right front back bottom top"
 
-rich_fby.Patch.front.BCPressure.Type = 'DirEquilRefPatch'
-rich_fby.Patch.front.BCPressure.Cycle = 'constant'
-rich_fby.Patch.front.BCPressure.RefGeom = 'domain'
-rich_fby.Patch.front.BCPressure.RefPatch = 'bottom'
+rich_fby.Patch.front.BCPressure.Type = "DirEquilRefPatch"
+rich_fby.Patch.front.BCPressure.Cycle = "constant"
+rich_fby.Patch.front.BCPressure.RefGeom = "domain"
+rich_fby.Patch.front.BCPressure.RefPatch = "bottom"
 rich_fby.Patch.front.BCPressure.alltime.Value = 11.0
 
-rich_fby.Patch.back.BCPressure.Type = 'DirEquilRefPatch'
-rich_fby.Patch.back.BCPressure.Cycle = 'constant'
-rich_fby.Patch.back.BCPressure.RefGeom = 'domain'
-rich_fby.Patch.back.BCPressure.RefPatch = 'bottom'
+rich_fby.Patch.back.BCPressure.Type = "DirEquilRefPatch"
+rich_fby.Patch.back.BCPressure.Cycle = "constant"
+rich_fby.Patch.back.BCPressure.RefGeom = "domain"
+rich_fby.Patch.back.BCPressure.RefPatch = "bottom"
 rich_fby.Patch.back.BCPressure.alltime.Value = 15.0
 
-rich_fby.Patch.left.BCPressure.Type = 'FluxConst'
-rich_fby.Patch.left.BCPressure.Cycle = 'constant'
+rich_fby.Patch.left.BCPressure.Type = "FluxConst"
+rich_fby.Patch.left.BCPressure.Cycle = "constant"
 rich_fby.Patch.left.BCPressure.alltime.Value = 0.0
 
-rich_fby.Patch.right.BCPressure.Type = 'FluxConst'
-rich_fby.Patch.right.BCPressure.Cycle = 'constant'
+rich_fby.Patch.right.BCPressure.Type = "FluxConst"
+rich_fby.Patch.right.BCPressure.Cycle = "constant"
 rich_fby.Patch.right.BCPressure.alltime.Value = 0.0
 
-rich_fby.Patch.bottom.BCPressure.Type = 'FluxConst'
-rich_fby.Patch.bottom.BCPressure.Cycle = 'constant'
+rich_fby.Patch.bottom.BCPressure.Type = "FluxConst"
+rich_fby.Patch.bottom.BCPressure.Cycle = "constant"
 rich_fby.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-rich_fby.Patch.top.BCPressure.Type = 'FluxConst'
-rich_fby.Patch.top.BCPressure.Cycle = 'constant'
+rich_fby.Patch.top.BCPressure.Type = "FluxConst"
+rich_fby.Patch.top.BCPressure.Cycle = "constant"
 rich_fby.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich_fby.TopoSlopesX.Type = 'Constant'
-rich_fby.TopoSlopesX.GeomNames = 'domain'
+rich_fby.TopoSlopesX.Type = "Constant"
+rich_fby.TopoSlopesX.GeomNames = "domain"
 rich_fby.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich_fby.TopoSlopesY.Type = 'Constant'
-rich_fby.TopoSlopesY.GeomNames = 'domain'
+rich_fby.TopoSlopesY.Type = "Constant"
+rich_fby.TopoSlopesY.GeomNames = "domain"
 rich_fby.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich_fby.Mannings.Type = 'Constant'
-rich_fby.Mannings.GeomNames = 'domain'
-rich_fby.Mannings.Geom.domain.Value = 0.
+rich_fby.Mannings.Type = "Constant"
+rich_fby.Mannings.GeomNames = "domain"
+rich_fby.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich_fby.ICPressure.Type = 'HydroStaticPatch'
-rich_fby.ICPressure.GeomNames = 'domain'
+rich_fby.ICPressure.Type = "HydroStaticPatch"
+rich_fby.ICPressure.GeomNames = "domain"
 rich_fby.Geom.domain.ICPressure.Value = 13.0
-rich_fby.Geom.domain.ICPressure.RefGeom = 'domain'
-rich_fby.Geom.domain.ICPressure.RefPatch = 'bottom'
+rich_fby.Geom.domain.ICPressure.RefGeom = "domain"
+rich_fby.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fby.PhaseSources.water.Type = 'Constant'
-rich_fby.PhaseSources.water.GeomNames = 'domain'
+rich_fby.PhaseSources.water.Type = "Constant"
+rich_fby.PhaseSources.water.GeomNames = "domain"
 rich_fby.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fby.KnownSolution = 'NoKnownSolution'
+rich_fby.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich_fby.Solver = 'Richards'
+rich_fby.Solver = "Richards"
 rich_fby.Solver.MaxIter = 50000
 
 rich_fby.Solver.Nonlinear.MaxIter = 100
 rich_fby.Solver.Nonlinear.ResidualTol = 1e-6
-rich_fby.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+rich_fby.Solver.Nonlinear.EtaChoice = "EtaConstant"
 rich_fby.Solver.Nonlinear.EtaValue = 1e-2
 rich_fby.Solver.Nonlinear.UseJacobian = True
 
@@ -298,10 +301,33 @@
 
 rich_fby.Solver.Linear.KrylovDimension = 100
 
-rich_fby.Solver.Linear.Preconditioner = 'PFMG'
+rich_fby.Solver.Linear.Preconditioner = "PFMG"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-rich_fby.run()
+# -----------------------------------------------------------------------------
+
+rich_fby.run(working_directory=new_output_dir_name)
+passed = True
+for i in range(11):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/richards_box_proctest.py b/test/python/richards_box_proctest.py
index b36cc8fbd..9b50b6f4a 100644
--- a/test/python/richards_box_proctest.py
+++ b/test/python/richards_box_proctest.py
@@ -1,14 +1,17 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # This runs a test case with the Richards' solver
 # with simple flow domains, like a wall or a fault.
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
-rbp = Run("rbp", __file__)
+run_name = "richards_ptest"
+rbp = Run(run_name, __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rbp.FileVersion = 4
 
@@ -16,9 +19,9 @@
 rbp.Process.Topology.Q = 1
 rbp.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rbp.ComputationalGrid.Lower.X = 0.0
 rbp.ComputationalGrid.Lower.Y = 0.0
@@ -32,22 +35,22 @@
 rbp.ComputationalGrid.NY = 50
 rbp.ComputationalGrid.NZ = 20
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rbp.GeomInput.Names = 'domain_input'
+rbp.GeomInput.Names = "domain_input"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rbp.GeomInput.domain_input.InputType = 'Box'
-rbp.GeomInput.domain_input.GeomName = 'domain'
+rbp.GeomInput.domain_input.InputType = "Box"
+rbp.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rbp.Geom.domain.Lower.X = 0.0
 rbp.Geom.domain.Lower.Y = 0.0
@@ -57,221 +60,221 @@
 rbp.Geom.domain.Upper.Y = 20.0
 rbp.Geom.domain.Upper.Z = 20.0
 
-rbp.Geom.domain.Patches = 'left right front back bottom top'
+rbp.Geom.domain.Patches = "left right front back bottom top"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbp.Geom.Perm.Names = 'domain'
-rbp.Geom.domain.Perm.Type = 'Constant'
+rbp.Geom.Perm.Names = "domain"
+rbp.Geom.domain.Perm.Type = "Constant"
 rbp.Geom.domain.Perm.Value = 1.0
 
-rbp.Perm.TensorType = 'TensorByGeom'
+rbp.Perm.TensorType = "TensorByGeom"
 
-rbp.Geom.Perm.TensorByGeom.Names = 'domain'
+rbp.Geom.Perm.TensorByGeom.Names = "domain"
 
 rbp.Geom.domain.Perm.TensorValX = 1.0
 rbp.Geom.domain.Perm.TensorValY = 1.0
 rbp.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbp.SpecificStorage.Type = 'Constant'
-rbp.SpecificStorage.GeomNames = 'domain'
+rbp.SpecificStorage.Type = "Constant"
+rbp.SpecificStorage.GeomNames = "domain"
 rbp.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbp.Phase.Names = 'water'
+rbp.Phase.Names = "water"
 
-rbp.Phase.water.Density.Type = 'Constant'
+rbp.Phase.water.Density.Type = "Constant"
 rbp.Phase.water.Density.Value = 1.0
 
-rbp.Phase.water.Viscosity.Type = 'Constant'
+rbp.Phase.water.Viscosity.Type = "Constant"
 rbp.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbp.Contaminants.Names = ''
+rbp.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbp.Geom.Retardation.GeomNames = ''
+rbp.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 rbp.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbp.TimingInfo.BaseUnit = 10.
+rbp.TimingInfo.BaseUnit = 10.0
 rbp.TimingInfo.StartCount = 0
 rbp.TimingInfo.StartTime = 0.0
 rbp.TimingInfo.StopTime = 100.0
 rbp.TimingInfo.DumpInterval = 10.0
-rbp.TimeStep.Type = 'Constant'
+rbp.TimeStep.Type = "Constant"
 rbp.TimeStep.Value = 10.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbp.Geom.Porosity.GeomNames = 'domain'
-rbp.Geom.domain.Porosity.Type = 'Constant'
+rbp.Geom.Porosity.GeomNames = "domain"
+rbp.Geom.domain.Porosity.Type = "Constant"
 rbp.Geom.domain.Porosity.Value = 0.25
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbp.Domain.GeomName = 'domain'
+rbp.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbp.Phase.RelPerm.Type = 'VanGenuchten'
-rbp.Phase.RelPerm.GeomNames = 'domain'
+rbp.Phase.RelPerm.Type = "VanGenuchten"
+rbp.Phase.RelPerm.GeomNames = "domain"
 rbp.Geom.domain.RelPerm.Alpha = 2.0
 rbp.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rbp.Phase.Saturation.Type = 'VanGenuchten'
-rbp.Phase.Saturation.GeomNames = 'domain'
+rbp.Phase.Saturation.Type = "VanGenuchten"
+rbp.Phase.Saturation.GeomNames = "domain"
 rbp.Geom.domain.Saturation.Alpha = 2.0
 rbp.Geom.domain.Saturation.N = 2.0
 rbp.Geom.domain.Saturation.SRes = 0.1
 rbp.Geom.domain.Saturation.SSat = 1.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Flow Barrier in X between cells 10 and 11 in all Z
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rbp.Solver.Nonlinear.FlowBarrierX = False
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbp.Wells.Names = ''
+rbp.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbp.Cycle.Names = 'constant'
-rbp.Cycle.constant.Names = 'alltime'
+rbp.Cycle.Names = "constant"
+rbp.Cycle.constant.Names = "alltime"
 rbp.Cycle.constant.alltime.Length = 1
 rbp.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbp.BCPressure.PatchNames = 'left right front back bottom top'
+rbp.BCPressure.PatchNames = "left right front back bottom top"
 
-rbp.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-rbp.Patch.left.BCPressure.Cycle = 'constant'
-rbp.Patch.left.BCPressure.RefGeom = 'domain'
-rbp.Patch.left.BCPressure.RefPatch = 'bottom'
+rbp.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+rbp.Patch.left.BCPressure.Cycle = "constant"
+rbp.Patch.left.BCPressure.RefGeom = "domain"
+rbp.Patch.left.BCPressure.RefPatch = "bottom"
 rbp.Patch.left.BCPressure.alltime.Value = 11.0
 
-rbp.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-rbp.Patch.right.BCPressure.Cycle = 'constant'
-rbp.Patch.right.BCPressure.RefGeom = 'domain'
-rbp.Patch.right.BCPressure.RefPatch = 'bottom'
+rbp.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+rbp.Patch.right.BCPressure.Cycle = "constant"
+rbp.Patch.right.BCPressure.RefGeom = "domain"
+rbp.Patch.right.BCPressure.RefPatch = "bottom"
 rbp.Patch.right.BCPressure.alltime.Value = 15.0
 
-rbp.Patch.front.BCPressure.Type = 'FluxConst'
-rbp.Patch.front.BCPressure.Cycle = 'constant'
+rbp.Patch.front.BCPressure.Type = "FluxConst"
+rbp.Patch.front.BCPressure.Cycle = "constant"
 rbp.Patch.front.BCPressure.alltime.Value = 0.0
 
-rbp.Patch.back.BCPressure.Type = 'FluxConst'
-rbp.Patch.back.BCPressure.Cycle = 'constant'
+rbp.Patch.back.BCPressure.Type = "FluxConst"
+rbp.Patch.back.BCPressure.Cycle = "constant"
 rbp.Patch.back.BCPressure.alltime.Value = 0.0
 
-rbp.Patch.bottom.BCPressure.Type = 'FluxConst'
-rbp.Patch.bottom.BCPressure.Cycle = 'constant'
+rbp.Patch.bottom.BCPressure.Type = "FluxConst"
+rbp.Patch.bottom.BCPressure.Cycle = "constant"
 rbp.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-rbp.Patch.top.BCPressure.Type = 'FluxConst'
-rbp.Patch.top.BCPressure.Cycle = 'constant'
+rbp.Patch.top.BCPressure.Type = "FluxConst"
+rbp.Patch.top.BCPressure.Cycle = "constant"
 rbp.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rbp.TopoSlopesX.Type = 'Constant'
-rbp.TopoSlopesX.GeomNames = 'domain'
+rbp.TopoSlopesX.Type = "Constant"
+rbp.TopoSlopesX.GeomNames = "domain"
 rbp.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rbp.TopoSlopesY.Type = 'Constant'
-rbp.TopoSlopesY.GeomNames = 'domain'
+rbp.TopoSlopesY.Type = "Constant"
+rbp.TopoSlopesY.GeomNames = "domain"
 rbp.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rbp.Mannings.Type = 'Constant'
-rbp.Mannings.GeomNames = 'domain'
-rbp.Mannings.Geom.domain.Value = 0.
+rbp.Mannings.Type = "Constant"
+rbp.Mannings.GeomNames = "domain"
+rbp.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rbp.ICPressure.Type = 'HydroStaticPatch'
-rbp.ICPressure.GeomNames = 'domain'
+rbp.ICPressure.Type = "HydroStaticPatch"
+rbp.ICPressure.GeomNames = "domain"
 rbp.Geom.domain.ICPressure.Value = 13.0
-rbp.Geom.domain.ICPressure.RefGeom = 'domain'
-rbp.Geom.domain.ICPressure.RefPatch = 'bottom'
+rbp.Geom.domain.ICPressure.RefGeom = "domain"
+rbp.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbp.PhaseSources.water.Type = 'Constant'
-rbp.PhaseSources.water.GeomNames = 'domain'
+rbp.PhaseSources.water.Type = "Constant"
+rbp.PhaseSources.water.GeomNames = "domain"
 rbp.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbp.KnownSolution = 'NoKnownSolution'
+rbp.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbp.Solver = 'Richards'
+rbp.Solver = "Richards"
 rbp.Solver.MaxIter = 50000
 
 rbp.Solver.Nonlinear.MaxIter = 100
 rbp.Solver.Nonlinear.ResidualTol = 1e-6
-rbp.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+rbp.Solver.Nonlinear.EtaChoice = "EtaConstant"
 rbp.Solver.Nonlinear.EtaValue = 1e-2
 rbp.Solver.Nonlinear.UseJacobian = True
 
@@ -279,12 +282,37 @@
 
 rbp.Solver.Linear.KrylovDimension = 100
 
-rbp.Solver.Linear.Preconditioner = 'PFMG'
+rbp.Solver.Linear.Preconditioner = "PFMG"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/rbp')
-mkdir(dir_name)
-rbp.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+correct_output_dir_name = get_absolute_path("../correct_output")
+new_output_dir_name = get_absolute_path("test_output/richards_ptest")
+mkdir(new_output_dir_name)
+
+rbp.run(working_directory=new_output_dir_name)
+passed = True
+for i in range(11):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/richards_box_proctest_vardz.py b/test/python/richards_box_proctest_vardz.py
index 0d37f135e..d7ccf3303 100644
--- a/test/python/richards_box_proctest_vardz.py
+++ b/test/python/richards_box_proctest_vardz.py
@@ -1,28 +1,38 @@
-#----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 # This runs a test case with the Richards' solver
 # with a simple flow domain and different BCs on the top.
 # The domain geometry is purposefully smaller than the computational grid
 # making more than 1/2 the domain inactive in Y.  When run with topology
 # 1 2 1 this will test PF behavior for inactive processors, for different BCs
 # and solver configurations.
-#----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 
+import sys, argparse
 from parflow import Run
-from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.fs import mkdir, get_absolute_path, rm
+from parflow.tools.compare import pf_test_file
 
-rbpv = Run("rbpv", __file__)
+run_name = "richards_ptest_vdz"
+rbpv = Run(run_name, __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rbpv.FileVersion = 4
 
-rbpv.Process.Topology.P = 1
-rbpv.Process.Topology.Q = 1
-rbpv.Process.Topology.R = 1
 
-#---------------------------------------------------------
+parser = argparse.ArgumentParser()
+parser.add_argument("-p", "--p", default=1)
+parser.add_argument("-q", "--q", default=1)
+parser.add_argument("-r", "--r", default=1)
+args = parser.parse_args()
+
+rbpv.Process.Topology.P = args.p
+rbpv.Process.Topology.Q = args.q
+rbpv.Process.Topology.R = args.r
+
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rbpv.ComputationalGrid.Lower.X = 0.0
 rbpv.ComputationalGrid.Lower.Y = 0.0
@@ -36,22 +46,22 @@
 rbpv.ComputationalGrid.NY = 50
 rbpv.ComputationalGrid.NZ = 20
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rbpv.GeomInput.Names = 'domain_input'
+rbpv.GeomInput.Names = "domain_input"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rbpv.GeomInput.domain_input.InputType = 'Box'
-rbpv.GeomInput.domain_input.GeomName = 'domain'
+rbpv.GeomInput.domain_input.InputType = "Box"
+rbpv.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rbpv.Geom.domain.Lower.X = 0.0
 rbpv.Geom.domain.Lower.Y = 0.0
@@ -61,15 +71,15 @@
 rbpv.Geom.domain.Upper.Y = 20.0
 rbpv.Geom.domain.Upper.Z = 20.0
 
-rbpv.Geom.domain.Patches = 'left right front back bottom top'
+rbpv.Geom.domain.Patches = "left right front back bottom top"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # variable dz assignments
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rbpv.Solver.Nonlinear.VariableDz = True
-rbpv.dzScale.GeomNames = 'domain'
-rbpv.dzScale.Type = 'nzList'
+rbpv.dzScale.GeomNames = "domain"
+rbpv.dzScale.Type = "nzList"
 rbpv.dzScale.nzListNumber = 20
 rbpv.Cell._0.dzScale.Value = 1.0
 rbpv.Cell._1.dzScale.Value = 1.0
@@ -92,225 +102,225 @@
 rbpv.Cell._18.dzScale.Value = 1.0
 rbpv.Cell._19.dzScale.Value = 0.05
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbpv.Geom.Perm.Names = 'domain'
-rbpv.Geom.domain.Perm.Type = 'Constant'
+rbpv.Geom.Perm.Names = "domain"
+rbpv.Geom.domain.Perm.Type = "Constant"
 rbpv.Geom.domain.Perm.Value = 1.0
 
-rbpv.Perm.TensorType = 'TensorByGeom'
+rbpv.Perm.TensorType = "TensorByGeom"
 
-rbpv.Geom.Perm.TensorByGeom.Names = 'domain'
+rbpv.Geom.Perm.TensorByGeom.Names = "domain"
 
 rbpv.Geom.domain.Perm.TensorValX = 1.0
 rbpv.Geom.domain.Perm.TensorValY = 1.0
 rbpv.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbpv.SpecificStorage.Type = 'Constant'
-rbpv.SpecificStorage.GeomNames = 'domain'
+rbpv.SpecificStorage.Type = "Constant"
+rbpv.SpecificStorage.GeomNames = "domain"
 rbpv.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbpv.Phase.Names = 'water'
+rbpv.Phase.Names = "water"
 
-rbpv.Phase.water.Density.Type = 'Constant'
+rbpv.Phase.water.Density.Type = "Constant"
 rbpv.Phase.water.Density.Value = 1.0
 
-rbpv.Phase.water.Viscosity.Type = 'Constant'
+rbpv.Phase.water.Viscosity.Type = "Constant"
 rbpv.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbpv.Contaminants.Names = ''
+rbpv.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbpv.Geom.Retardation.GeomNames = ''
+rbpv.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 rbpv.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbpv.TimingInfo.BaseUnit = 10.
+rbpv.TimingInfo.BaseUnit = 10.0
 rbpv.TimingInfo.StartCount = 0
 rbpv.TimingInfo.StartTime = 0.0
 rbpv.TimingInfo.StopTime = 100.0
 rbpv.TimingInfo.DumpInterval = 10.0
-rbpv.TimeStep.Type = 'Constant'
+rbpv.TimeStep.Type = "Constant"
 rbpv.TimeStep.Value = 10.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbpv.Geom.Porosity.GeomNames = 'domain'
-rbpv.Geom.domain.Porosity.Type = 'Constant'
+rbpv.Geom.Porosity.GeomNames = "domain"
+rbpv.Geom.domain.Porosity.Type = "Constant"
 rbpv.Geom.domain.Porosity.Value = 0.25
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbpv.Domain.GeomName = 'domain'
+rbpv.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbpv.Phase.RelPerm.Type = 'VanGenuchten'
-rbpv.Phase.RelPerm.GeomNames = 'domain'
+rbpv.Phase.RelPerm.Type = "VanGenuchten"
+rbpv.Phase.RelPerm.GeomNames = "domain"
 rbpv.Geom.domain.RelPerm.Alpha = 2.0
 rbpv.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rbpv.Phase.Saturation.Type = 'VanGenuchten'
-rbpv.Phase.Saturation.GeomNames = 'domain'
+rbpv.Phase.Saturation.Type = "VanGenuchten"
+rbpv.Phase.Saturation.GeomNames = "domain"
 rbpv.Geom.domain.Saturation.Alpha = 2.0
 rbpv.Geom.domain.Saturation.N = 2.0
 rbpv.Geom.domain.Saturation.SRes = 0.1
 rbpv.Geom.domain.Saturation.SSat = 1.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # No Flow Barrier
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rbpv.Solver.Nonlinear.FlowBarrierX = False
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbpv.Wells.Names = ''
+rbpv.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbpv.Cycle.Names = 'constant'
-rbpv.Cycle.constant.Names = 'alltime'
+rbpv.Cycle.Names = "constant"
+rbpv.Cycle.constant.Names = "alltime"
 rbpv.Cycle.constant.alltime.Length = 1
 rbpv.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbpv.BCPressure.PatchNames = 'left right front back bottom top'
+rbpv.BCPressure.PatchNames = "left right front back bottom top"
 
-rbpv.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-rbpv.Patch.left.BCPressure.Cycle = 'constant'
-rbpv.Patch.left.BCPressure.RefGeom = 'domain'
-rbpv.Patch.left.BCPressure.RefPatch = 'bottom'
+rbpv.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+rbpv.Patch.left.BCPressure.Cycle = "constant"
+rbpv.Patch.left.BCPressure.RefGeom = "domain"
+rbpv.Patch.left.BCPressure.RefPatch = "bottom"
 rbpv.Patch.left.BCPressure.alltime.Value = 11.0
 
-rbpv.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-rbpv.Patch.right.BCPressure.Cycle = 'constant'
-rbpv.Patch.right.BCPressure.RefGeom = 'domain'
-rbpv.Patch.right.BCPressure.RefPatch = 'bottom'
+rbpv.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+rbpv.Patch.right.BCPressure.Cycle = "constant"
+rbpv.Patch.right.BCPressure.RefGeom = "domain"
+rbpv.Patch.right.BCPressure.RefPatch = "bottom"
 rbpv.Patch.right.BCPressure.alltime.Value = 15.0
 
-rbpv.Patch.front.BCPressure.Type = 'FluxConst'
-rbpv.Patch.front.BCPressure.Cycle = 'constant'
+rbpv.Patch.front.BCPressure.Type = "FluxConst"
+rbpv.Patch.front.BCPressure.Cycle = "constant"
 rbpv.Patch.front.BCPressure.alltime.Value = 0.0
 
-rbpv.Patch.back.BCPressure.Type = 'FluxConst'
-rbpv.Patch.back.BCPressure.Cycle = 'constant'
+rbpv.Patch.back.BCPressure.Type = "FluxConst"
+rbpv.Patch.back.BCPressure.Cycle = "constant"
 rbpv.Patch.back.BCPressure.alltime.Value = 0.0
 
-rbpv.Patch.bottom.BCPressure.Type = 'FluxConst'
-rbpv.Patch.bottom.BCPressure.Cycle = 'constant'
+rbpv.Patch.bottom.BCPressure.Type = "FluxConst"
+rbpv.Patch.bottom.BCPressure.Cycle = "constant"
 rbpv.Patch.bottom.BCPressure.alltime.Value = 0.0
 
 # used to cycle different BCs on the top of the domain, even with no
 # overland flow
-rbpv.Patch.top.BCPressure.Type = 'FluxConst'
-rbpv.Patch.top.BCPressure.Type = 'OverlandFlow'
-rbpv.Patch.top.BCPressure.Type = 'OverlandKinematic'
+rbpv.Patch.top.BCPressure.Type = "FluxConst"
+rbpv.Patch.top.BCPressure.Type = "OverlandFlow"
+rbpv.Patch.top.BCPressure.Type = "OverlandKinematic"
 
-rbpv.Patch.top.BCPressure.Cycle = 'constant'
+rbpv.Patch.top.BCPressure.Cycle = "constant"
 rbpv.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rbpv.TopoSlopesX.Type = 'Constant'
-rbpv.TopoSlopesX.GeomNames = 'domain'
+rbpv.TopoSlopesX.Type = "Constant"
+rbpv.TopoSlopesX.GeomNames = ""
 rbpv.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rbpv.TopoSlopesY.Type = 'Constant'
-rbpv.TopoSlopesY.GeomNames = 'domain'
+rbpv.TopoSlopesY.Type = "Constant"
+rbpv.TopoSlopesY.GeomNames = ""
 rbpv.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rbpv.Mannings.Type = 'Constant'
-rbpv.Mannings.GeomNames = 'domain'
-rbpv.Mannings.Geom.domain.Value = 0.
+rbpv.Mannings.Type = "Constant"
+rbpv.Mannings.GeomNames = ""
+rbpv.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rbpv.ICPressure.Type = 'HydroStaticPatch'
-rbpv.ICPressure.GeomNames = 'domain'
+rbpv.ICPressure.Type = "HydroStaticPatch"
+rbpv.ICPressure.GeomNames = "domain"
 rbpv.Geom.domain.ICPressure.Value = 13.0
-rbpv.Geom.domain.ICPressure.RefGeom = 'domain'
-rbpv.Geom.domain.ICPressure.RefPatch = 'bottom'
+rbpv.Geom.domain.ICPressure.RefGeom = "domain"
+rbpv.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbpv.PhaseSources.water.Type = 'Constant'
-rbpv.PhaseSources.water.GeomNames = 'domain'
+rbpv.PhaseSources.water.Type = "Constant"
+rbpv.PhaseSources.water.GeomNames = "domain"
 rbpv.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbpv.KnownSolution = 'NoKnownSolution'
+rbpv.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rbpv.Solver = 'Richards'
+rbpv.Solver = "Richards"
 rbpv.Solver.MaxIter = 50000
 
 rbpv.Solver.Nonlinear.MaxIter = 100
 rbpv.Solver.Nonlinear.ResidualTol = 1e-7
 
-rbpv.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+rbpv.Solver.Nonlinear.EtaChoice = "EtaConstant"
 rbpv.Solver.Nonlinear.EtaValue = 1e-2
 
 # used to test analytical and FD jacobian combinations
@@ -321,14 +331,40 @@
 rbpv.Solver.Linear.KrylovDimension = 100
 
 # used to test different linear preconditioners
-rbpv.Solver.Linear.Preconditioner = 'PFMG'
+rbpv.Solver.Linear.Preconditioner = "PFMG"
 
 rbpv.UseClustering = False
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/rbpv')
-mkdir(dir_name)
-rbpv.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+correct_output_dir_name = get_absolute_path("../correct_output")
+new_output_dir_name = get_absolute_path("test_output/richards_ptest_vdz")
+mkdir(new_output_dir_name)
+
+rbpv.run(working_directory=new_output_dir_name)
+passed = True
+for i in range(11):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+rm(new_output_dir_name)
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/richards_hydrostatic_equilibrium.py b/test/python/richards_hydrostatic_equilibrium.py
index 4b43aca55..1d25b0ed8 100644
--- a/test/python/richards_hydrostatic_equilibrium.py
+++ b/test/python/richards_hydrostatic_equilibrium.py
@@ -1,16 +1,19 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # This runs a test case with the Richards' solver
 # in hydrostatic equilibrium.  As such the solution
 # should not change over time and should not
 # take any solver iterations.
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
-rich = Run("richards_hydrostatic_equilibrium", __file__)
+run_name = "richards_hydrostatic_equalibrium"
+rich = Run(run_name, __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rich.FileVersion = 4
 
@@ -18,9 +21,9 @@
 rich.Process.Topology.Q = 1
 rich.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rich.ComputationalGrid.Lower.X = 0.0
 rich.ComputationalGrid.Lower.Y = 0.0
@@ -34,22 +37,22 @@
 rich.ComputationalGrid.NY = 20
 rich.ComputationalGrid.NZ = 10
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich.GeomInput.Names = 'domain_input background_input'
+rich.GeomInput.Names = "domain_input background_input"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich.GeomInput.domain_input.InputType = 'Box'
-rich.GeomInput.domain_input.GeomName = 'domain'
+rich.GeomInput.domain_input.InputType = "Box"
+rich.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rich.Geom.domain.Lower.X = 0.0
 rich.Geom.domain.Lower.Y = 0.0
@@ -59,18 +62,18 @@
 rich.Geom.domain.Upper.Y = 19.0
 rich.Geom.domain.Upper.Z = 3.0
 
-rich.Geom.domain.Patches = 'left right front back bottom top'
+rich.Geom.domain.Patches = "left right front back bottom top"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich.GeomInput.background_input.InputType = 'Box'
-rich.GeomInput.background_input.GeomName = 'background'
+rich.GeomInput.background_input.InputType = "Box"
+rich.GeomInput.background_input.GeomName = "background"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rich.Geom.background.Lower.X = -99999999.0
 rich.Geom.background.Lower.Y = -99999999.0
@@ -80,227 +83,267 @@
 rich.Geom.background.Upper.Y = 99999999.0
 rich.Geom.background.Upper.Z = 99999999.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich.Geom.Perm.Names = 'background'
-rich.Geom.background.Perm.Type = 'Constant'
+rich.Geom.Perm.Names = "background"
+rich.Geom.background.Perm.Type = "Constant"
 rich.Geom.background.Perm.Value = 4.0
 
-rich.Perm.TensorType = 'TensorByGeom'
+rich.Perm.TensorType = "TensorByGeom"
 
-rich.Geom.Perm.TensorByGeom.Names = 'background'
+rich.Geom.Perm.TensorByGeom.Names = "background"
 
 rich.Geom.background.Perm.TensorValX = 1.0
 rich.Geom.background.Perm.TensorValY = 1.0
 rich.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich.SpecificStorage.Type = 'Constant'
-rich.SpecificStorage.GeomNames = 'background'
+rich.SpecificStorage.Type = "Constant"
+rich.SpecificStorage.GeomNames = "background"
 rich.Geom.background.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich.Phase.Names = 'water'
+rich.Phase.Names = "water"
 
-rich.Phase.water.Density.Type = 'Constant'
+rich.Phase.water.Density.Type = "Constant"
 rich.Phase.water.Density.Value = 1.0
 
-rich.Phase.water.Viscosity.Type = 'Constant'
+rich.Phase.water.Viscosity.Type = "Constant"
 rich.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich.Contaminants.Names = ''
+rich.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich.Geom.Retardation.GeomNames = ''
+rich.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 rich.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 rich.TimingInfo.BaseUnit = 0.5
 rich.TimingInfo.StartCount = 0
 rich.TimingInfo.StartTime = 0.0
 rich.TimingInfo.StopTime = 1.5
 rich.TimingInfo.DumpInterval = -1
-rich.TimeStep.Type = 'Constant'
+rich.TimeStep.Type = "Constant"
 rich.TimeStep.Value = 0.5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich.Geom.Porosity.GeomNames = 'background'
-rich.Geom.background.Porosity.Type = 'Constant'
+rich.Geom.Porosity.GeomNames = "background"
+rich.Geom.background.Porosity.Type = "Constant"
 rich.Geom.background.Porosity.Value = 0.15
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich.Domain.GeomName = 'domain'
+rich.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich.Phase.RelPerm.Type = 'VanGenuchten'
-rich.Phase.RelPerm.GeomNames = 'background'
+rich.Phase.RelPerm.Type = "VanGenuchten"
+rich.Phase.RelPerm.GeomNames = "background"
 rich.Geom.background.RelPerm.Alpha = 2.0
 rich.Geom.background.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich.Phase.Saturation.Type = 'VanGenuchten'
-rich.Phase.Saturation.GeomNames = 'background'
+rich.Phase.Saturation.Type = "VanGenuchten"
+rich.Phase.Saturation.GeomNames = "background"
 rich.Geom.background.Saturation.Alpha = 2.0
 rich.Geom.background.Saturation.N = 2.0
 rich.Geom.background.Saturation.SRes = 0.0
 rich.Geom.background.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich.Wells.Names = ''
+rich.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich.Cycle.Names = 'constant'
-rich.Cycle.constant.Names = 'alltime'
+rich.Cycle.Names = "constant"
+rich.Cycle.constant.Names = "alltime"
 rich.Cycle.constant.alltime.Length = 1
 rich.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich.BCPressure.PatchNames = 'left right front back bottom top'
+rich.BCPressure.PatchNames = "left right front back bottom top"
 
-rich.Patch.front.BCPressure.Type = 'DirEquilRefPatch'
-rich.Patch.front.BCPressure.Cycle = 'constant'
-rich.Patch.front.BCPressure.RefGeom = 'domain'
-rich.Patch.front.BCPressure.RefPatch = 'bottom'
+rich.Patch.front.BCPressure.Type = "DirEquilRefPatch"
+rich.Patch.front.BCPressure.Cycle = "constant"
+rich.Patch.front.BCPressure.RefGeom = "domain"
+rich.Patch.front.BCPressure.RefPatch = "bottom"
 rich.Patch.front.BCPressure.alltime.Value = 1.0
 
-rich.Patch.back.BCPressure.Type = 'DirEquilRefPatch'
-rich.Patch.back.BCPressure.Cycle = 'constant'
-rich.Patch.back.BCPressure.RefGeom = 'domain'
-rich.Patch.back.BCPressure.RefPatch = 'bottom'
+rich.Patch.back.BCPressure.Type = "DirEquilRefPatch"
+rich.Patch.back.BCPressure.Cycle = "constant"
+rich.Patch.back.BCPressure.RefGeom = "domain"
+rich.Patch.back.BCPressure.RefPatch = "bottom"
 rich.Patch.back.BCPressure.alltime.Value = 1.0
 
-rich.Patch.left.BCPressure.Type = 'FluxConst'
-rich.Patch.left.BCPressure.Cycle = 'constant'
+rich.Patch.left.BCPressure.Type = "FluxConst"
+rich.Patch.left.BCPressure.Cycle = "constant"
 rich.Patch.left.BCPressure.alltime.Value = 0.0
 
-rich.Patch.right.BCPressure.Type = 'FluxConst'
-rich.Patch.right.BCPressure.Cycle = 'constant'
+rich.Patch.right.BCPressure.Type = "FluxConst"
+rich.Patch.right.BCPressure.Cycle = "constant"
 rich.Patch.right.BCPressure.alltime.Value = 0.0
 
-rich.Patch.bottom.BCPressure.Type = 'FluxConst'
-rich.Patch.bottom.BCPressure.Cycle = 'constant'
+rich.Patch.bottom.BCPressure.Type = "FluxConst"
+rich.Patch.bottom.BCPressure.Cycle = "constant"
 rich.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-rich.Patch.top.BCPressure.Type = 'FluxConst'
-rich.Patch.top.BCPressure.Cycle = 'constant'
+rich.Patch.top.BCPressure.Type = "FluxConst"
+rich.Patch.top.BCPressure.Cycle = "constant"
 rich.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich.TopoSlopesX.Type = 'Constant'
-rich.TopoSlopesX.GeomNames = 'domain'
+rich.TopoSlopesX.Type = "Constant"
+rich.TopoSlopesX.GeomNames = "domain"
 rich.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich.TopoSlopesY.Type = 'Constant'
-rich.TopoSlopesY.GeomNames = 'domain'
+rich.TopoSlopesY.Type = "Constant"
+rich.TopoSlopesY.GeomNames = "domain"
 rich.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-rich.Mannings.Type = 'Constant'
-rich.Mannings.GeomNames = 'domain'
-rich.Mannings.Geom.domain.Value = 0.
+rich.Mannings.Type = "Constant"
+rich.Mannings.GeomNames = "domain"
+rich.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-rich.ICPressure.Type = 'HydroStaticPatch'
-rich.ICPressure.GeomNames = 'domain'
+rich.ICPressure.Type = "HydroStaticPatch"
+rich.ICPressure.GeomNames = "domain"
 rich.Geom.domain.ICPressure.Value = 1.0
-rich.Geom.domain.ICPressure.RefGeom = 'domain'
-rich.Geom.domain.ICPressure.RefPatch = 'bottom'
+rich.Geom.domain.ICPressure.RefGeom = "domain"
+rich.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich.PhaseSources.water.Type = 'Constant'
-rich.PhaseSources.water.GeomNames = 'background'
+rich.PhaseSources.water.Type = "Constant"
+rich.PhaseSources.water.GeomNames = "background"
 rich.PhaseSources.water.Geom.background.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich.KnownSolution = 'NoKnownSolution'
+rich.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-rich.Solver = 'Richards'
+rich.Solver = "Richards"
 rich.Solver.MaxIter = 50000
 
 rich.Solver.Nonlinear.MaxIter = 100
 rich.Solver.Nonlinear.ResidualTol = 1e-9
-rich.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+rich.Solver.Nonlinear.EtaChoice = "EtaConstant"
 rich.Solver.Nonlinear.EtaValue = 1e-2
 rich.Solver.Nonlinear.UseJacobian = True
 rich.Solver.Nonlinear.DerivativeEpsilon = 1e-9
 
 rich.Solver.Linear.KrylovDimension = 10
 
-rich.Solver.Linear.Preconditioner = 'MGSemi'
+rich.Solver.Linear.Preconditioner = "MGSemi"
 rich.Solver.Linear.Preconditioner.MGSemi.MaxIter = 10
 rich.Solver.Linear.Preconditioner.MGSemi.MaxLevels = 100
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/rich_heq')
-mkdir(dir_name)
-rich.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/richards_hydrostatic_equilibrium")
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+rich.run(working_directory=new_output_dir_name)
+
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+
+for i in range(3):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/small_domain.py b/test/python/small_domain.py
index 42a2c2833..939c9ab09 100644
--- a/test/python/small_domain.py
+++ b/test/python/small_domain.py
@@ -1,29 +1,32 @@
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 #  This is a 2D sloped problem w/ time varying input and topography
 #  it is used as a test of active/inactive efficiency
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import cp, mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
-small_domain = Run("small_domain", __file__)
+run_name = "small_domain"
+small_domain = Run(run_name, __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Copying solid file
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/small_domain')
-mkdir(dir_name)
+new_output_dir_name = get_absolute_path("test_output/small_domain")
+mkdir(new_output_dir_name)
 
-cp('$PF_SRC/test/input/small_domain.pfsol', dir_name)
+cp("$PF_SRC/test/input/crater2D.pfsol", new_output_dir_name)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Setting size for domain
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 size = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 small_domain.FileVersion = 4
 
@@ -31,21 +34,21 @@
 small_domain.Process.Topology.Q = 1
 small_domain.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 small_domain.ComputationalGrid.Lower.X = 0.0
 small_domain.ComputationalGrid.Lower.Y = 0.0
 small_domain.ComputationalGrid.Lower.Z = 0.0
 
-small_domain.ComputationalGrid.NX = 100*size
+small_domain.ComputationalGrid.NX = 100 * size
 small_domain.ComputationalGrid.NY = 1
-small_domain.ComputationalGrid.NZ = 100*size
+small_domain.ComputationalGrid.NZ = 100 * size
 
-UpperX = 400*size
+UpperX = 400 * size
 UpperY = 1.0
-UpperZ = 200*size
+UpperZ = 200 * size
 
 LowerX = small_domain.ComputationalGrid.Lower.X
 LowerY = small_domain.ComputationalGrid.Lower.Y
@@ -59,18 +62,18 @@
 small_domain.ComputationalGrid.DY = (UpperY - LowerY) / NY
 small_domain.ComputationalGrid.DZ = (UpperZ - LowerZ) / NZ
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-small_domain.GeomInput.Names = 'solidinput background'
+small_domain.GeomInput.Names = "solidinput background"
 
-small_domain.GeomInput.solidinput.InputType = 'SolidFile'
-small_domain.GeomInput.solidinput.GeomNames = 'domain'
-small_domain.GeomInput.solidinput.FileName = 'small_domain.pfsol'
+small_domain.GeomInput.solidinput.InputType = "SolidFile"
+small_domain.GeomInput.solidinput.GeomNames = "domain"
+small_domain.GeomInput.solidinput.FileName = "crater2D.pfsol"
 
-small_domain.GeomInput.background.InputType = 'Box'
-small_domain.GeomInput.background.GeomName = 'background'
+small_domain.GeomInput.background.InputType = "Box"
+small_domain.GeomInput.background.GeomName = "background"
 
 small_domain.Geom.background.Lower.X = -99999999.0
 small_domain.Geom.background.Lower.Y = -99999999.0
@@ -79,224 +82,226 @@
 small_domain.Geom.background.Upper.Y = 99999999.0
 small_domain.Geom.background.Upper.Z = 99999999.0
 
-small_domain.Geom.domain.Patches = 'infiltration z_upper x_lower y_lower x_upper y_upper z_lower'
+small_domain.Geom.domain.Patches = (
+    "infiltration z_upper x_lower y_lower x_upper y_upper z_lower"
+)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-small_domain.Geom.Perm.Names = 'domain'
+small_domain.Geom.Perm.Names = "domain"
 
-small_domain.Geom.domain.Perm.Type = 'Constant'
+small_domain.Geom.domain.Perm.Type = "Constant"
 small_domain.Geom.domain.Perm.Value = 1.0
 
-small_domain.Perm.TensorType = 'TensorByGeom'
+small_domain.Perm.TensorType = "TensorByGeom"
 
-small_domain.Geom.Perm.TensorByGeom.Names = 'background'
+small_domain.Geom.Perm.TensorByGeom.Names = "background"
 
 small_domain.Geom.background.Perm.TensorValX = 1.0
 small_domain.Geom.background.Perm.TensorValY = 1.0
 small_domain.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-small_domain.SpecificStorage.Type = 'Constant'
-small_domain.SpecificStorage.GeomNames = 'domain'
+small_domain.SpecificStorage.Type = "Constant"
+small_domain.SpecificStorage.GeomNames = "domain"
 small_domain.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-small_domain.Phase.Names = 'water'
+small_domain.Phase.Names = "water"
 
-small_domain.Phase.water.Density.Type = 'Constant'
+small_domain.Phase.water.Density.Type = "Constant"
 small_domain.Phase.water.Density.Value = 1.0
 
-small_domain.Phase.water.Viscosity.Type = 'Constant'
+small_domain.Phase.water.Viscosity.Type = "Constant"
 small_domain.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-small_domain.Contaminants.Names = ''
+small_domain.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-small_domain.Geom.Retardation.GeomNames = ''
+small_domain.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 small_domain.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 small_domain.TimingInfo.BaseUnit = 1.0
 small_domain.TimingInfo.StartCount = 0
 small_domain.TimingInfo.StartTime = 0.0
-small_domain.TimingInfo.StopTime = 30.0*1
+small_domain.TimingInfo.StopTime = 30.0 * 1
 small_domain.TimingInfo.DumpInterval = 10
-small_domain.TimeStep.Type = 'Constant'
+small_domain.TimeStep.Type = "Constant"
 small_domain.TimeStep.Value = 10.0
 small_domain.TimingInfo.DumpAtEnd = True
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-small_domain.Geom.Porosity.GeomNames = 'domain'
-small_domain.Geom.domain.Porosity.Type = 'Constant'
+small_domain.Geom.Porosity.GeomNames = "domain"
+small_domain.Geom.domain.Porosity.Type = "Constant"
 small_domain.Geom.domain.Porosity.Value = 0.3680
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-small_domain.Domain.GeomName = 'domain'
+small_domain.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-small_domain.Phase.RelPerm.Type = 'VanGenuchten'
-small_domain.Phase.RelPerm.GeomNames = 'domain'
+small_domain.Phase.RelPerm.Type = "VanGenuchten"
+small_domain.Phase.RelPerm.GeomNames = "domain"
 
 small_domain.Geom.domain.RelPerm.Alpha = 3.34
 small_domain.Geom.domain.RelPerm.N = 1.982
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-small_domain.Phase.Saturation.Type = 'VanGenuchten'
-small_domain.Phase.Saturation.GeomNames = 'domain'
+small_domain.Phase.Saturation.Type = "VanGenuchten"
+small_domain.Phase.Saturation.GeomNames = "domain"
 
 small_domain.Geom.domain.Saturation.Alpha = 3.34
 small_domain.Geom.domain.Saturation.N = 1.982
 small_domain.Geom.domain.Saturation.SRes = 0.2771
 small_domain.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-small_domain.Wells.Names = ''
+small_domain.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-small_domain.Cycle.Names = 'constant onoff'
-small_domain.Cycle.constant.Names = 'alltime'
+small_domain.Cycle.Names = "constant onoff"
+small_domain.Cycle.constant.Names = "alltime"
 small_domain.Cycle.constant.alltime.Length = 1
 small_domain.Cycle.constant.Repeat = -1
 
-small_domain.Cycle.onoff.Names = 'on off'
+small_domain.Cycle.onoff.Names = "on off"
 small_domain.Cycle.onoff.on.Length = 10
 small_domain.Cycle.onoff.off.Length = 90
 small_domain.Cycle.onoff.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 small_domain.BCPressure.PatchNames = small_domain.Geom.domain.Patches
 
-small_domain.Patch.infiltration.BCPressure.Type = 'FluxConst'
-small_domain.Patch.infiltration.BCPressure.Cycle = 'constant'
+small_domain.Patch.infiltration.BCPressure.Type = "FluxConst"
+small_domain.Patch.infiltration.BCPressure.Cycle = "constant"
 small_domain.Patch.infiltration.BCPressure.alltime.Value = -0.10
 small_domain.Patch.infiltration.BCPressure.off.Value = 0.0
 
-small_domain.Patch.x_lower.BCPressure.Type = 'FluxConst'
-small_domain.Patch.x_lower.BCPressure.Cycle = 'constant'
+small_domain.Patch.x_lower.BCPressure.Type = "FluxConst"
+small_domain.Patch.x_lower.BCPressure.Cycle = "constant"
 small_domain.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-small_domain.Patch.y_lower.BCPressure.Type = 'FluxConst'
-small_domain.Patch.y_lower.BCPressure.Cycle = 'constant'
+small_domain.Patch.y_lower.BCPressure.Type = "FluxConst"
+small_domain.Patch.y_lower.BCPressure.Cycle = "constant"
 small_domain.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-small_domain.Patch.z_lower.BCPressure.Type = 'FluxConst'
-small_domain.Patch.z_lower.BCPressure.Cycle = 'constant'
+small_domain.Patch.z_lower.BCPressure.Type = "FluxConst"
+small_domain.Patch.z_lower.BCPressure.Cycle = "constant"
 small_domain.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-small_domain.Patch.x_upper.BCPressure.Type = 'FluxConst'
-small_domain.Patch.x_upper.BCPressure.Cycle = 'constant'
+small_domain.Patch.x_upper.BCPressure.Type = "FluxConst"
+small_domain.Patch.x_upper.BCPressure.Cycle = "constant"
 small_domain.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-small_domain.Patch.y_upper.BCPressure.Type = 'FluxConst'
-small_domain.Patch.y_upper.BCPressure.Cycle = 'constant'
+small_domain.Patch.y_upper.BCPressure.Type = "FluxConst"
+small_domain.Patch.y_upper.BCPressure.Cycle = "constant"
 small_domain.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-small_domain.Patch.z_upper.BCPressure.Type = 'FluxConst'
-small_domain.Patch.z_upper.BCPressure.Cycle = 'constant'
+small_domain.Patch.z_upper.BCPressure.Type = "FluxConst"
+small_domain.Patch.z_upper.BCPressure.Cycle = "constant"
 small_domain.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-small_domain.TopoSlopesX.Type = 'Constant'
-small_domain.TopoSlopesX.GeomNames = 'domain'
+small_domain.TopoSlopesX.Type = "Constant"
+small_domain.TopoSlopesX.GeomNames = "domain"
 small_domain.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-small_domain.TopoSlopesY.Type = 'Constant'
-small_domain.TopoSlopesY.GeomNames = 'domain'
+small_domain.TopoSlopesY.Type = "Constant"
+small_domain.TopoSlopesY.GeomNames = "domain"
 small_domain.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-small_domain.Mannings.Type = 'Constant'
-small_domain.Mannings.GeomNames = 'domain'
-small_domain.Mannings.Geom.domain.Value = 0.
+small_domain.Mannings.Type = "Constant"
+small_domain.Mannings.GeomNames = "domain"
+small_domain.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-small_domain.ICPressure.Type = 'HydroStaticPatch'
-small_domain.ICPressure.GeomNames = 'domain'
+small_domain.ICPressure.Type = "HydroStaticPatch"
+small_domain.ICPressure.GeomNames = "domain"
 
 small_domain.Geom.domain.ICPressure.Value = 1.0
-small_domain.Geom.domain.ICPressure.RefPatch = 'z_lower'
-small_domain.Geom.domain.ICPressure.RefGeom = 'domain'
+small_domain.Geom.domain.ICPressure.RefPatch = "z_lower"
+small_domain.Geom.domain.ICPressure.RefGeom = "domain"
 
 small_domain.Geom.infiltration.ICPressure.Value = 10.0
-small_domain.Geom.infiltration.ICPressure.RefPatch = 'infiltration'
-small_domain.Geom.infiltration.ICPressure.RefGeom = 'domain'
+small_domain.Geom.infiltration.ICPressure.RefPatch = "infiltration"
+small_domain.Geom.infiltration.ICPressure.RefGeom = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-small_domain.PhaseSources.water.Type = 'Constant'
-small_domain.PhaseSources.water.GeomNames = 'background'
+small_domain.PhaseSources.water.Type = "Constant"
+small_domain.PhaseSources.water.GeomNames = "background"
 small_domain.PhaseSources.water.Geom.background.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-small_domain.KnownSolution = 'NoKnownSolution'
+small_domain.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-small_domain.Solver = 'Richards'
+small_domain.Solver = "Richards"
 small_domain.Solver.MaxIter = 10000
 
 small_domain.Solver.Nonlinear.MaxIter = 15
@@ -309,12 +314,49 @@
 small_domain.Solver.Linear.KrylovDimension = 25
 small_domain.Solver.Linear.MaxRestarts = 2
 
-small_domain.Solver.Linear.Preconditioner = 'MGSemi'
+small_domain.Solver.Linear.Preconditioner = "MGSemi"
 small_domain.Solver.Linear.Preconditioner.MGSemi.MaxIter = 1
 small_domain.Solver.Linear.Preconditioner.MGSemi.MaxLevels = 100
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-small_domain.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+correct_output_dir_name = get_absolute_path("../correct_output")
+small_domain.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z", "porosity"]
+
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+for i in range(5):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/smg.py b/test/python/smg.py
index 671173642..eb94a5f8f 100644
--- a/test/python/smg.py
+++ b/test/python/smg.py
@@ -1,15 +1,18 @@
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 #  This runs the basic smg test case based off of default richards
 #  This run, as written in this input file, should take
 #  3 nonlinear iterations.
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 
+import sys
 from parflow import Run
-from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.fs import mkdir, get_absolute_path, rm
+from parflow.tools.compare import pf_test_file
 
-smg = Run("smg", __file__)
+run_name = "smg"
+smg = Run(run_name, __file__)
 
-#------------------------------------------------------------------
+# ------------------------------------------------------------------
 
 smg.FileVersion = 4
 
@@ -17,9 +20,9 @@
 smg.Process.Topology.Q = 1
 smg.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 smg.ComputationalGrid.Lower.X = -10.0
 smg.ComputationalGrid.Lower.Y = 10.0
@@ -33,22 +36,24 @@
 smg.ComputationalGrid.NY = 10
 smg.ComputationalGrid.NZ = 8
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-smg.GeomInput.Names = 'domain_input background_input source_region_input concen_region_input'
+smg.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-smg.GeomInput.domain_input.InputType = 'Box'
-smg.GeomInput.domain_input.GeomName = 'domain'
+smg.GeomInput.domain_input.InputType = "Box"
+smg.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 smg.Geom.domain.Lower.X = -10.0
 smg.Geom.domain.Lower.Y = 10.0
@@ -58,18 +63,18 @@
 smg.Geom.domain.Upper.Y = 170.0
 smg.Geom.domain.Upper.Z = 9.0
 
-smg.Geom.domain.Patches = 'left right front back bottom top'
+smg.Geom.domain.Patches = "left right front back bottom top"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-smg.GeomInput.background_input.InputType = 'Box'
-smg.GeomInput.background_input.GeomName = 'background'
+smg.GeomInput.background_input.InputType = "Box"
+smg.GeomInput.background_input.GeomName = "background"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 smg.Geom.background.Lower.X = -99999999.0
 smg.Geom.background.Lower.Y = -99999999.0
@@ -79,16 +84,16 @@
 smg.Geom.background.Upper.Y = 99999999.0
 smg.Geom.background.Upper.Z = 99999999.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-smg.GeomInput.source_region_input.InputType = 'Box'
-smg.GeomInput.source_region_input.GeomName = 'source_region'
+smg.GeomInput.source_region_input.InputType = "Box"
+smg.GeomInput.source_region_input.GeomName = "source_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 smg.Geom.source_region.Lower.X = 65.56
 smg.Geom.source_region.Lower.Y = 79.34
@@ -98,16 +103,16 @@
 smg.Geom.source_region.Upper.Y = 89.99
 smg.Geom.source_region.Upper.Z = 5.5
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-smg.GeomInput.concen_region_input.InputType = 'Box'
-smg.GeomInput.concen_region_input.GeomName = 'concen_region'
+smg.GeomInput.concen_region_input.InputType = "Box"
+smg.GeomInput.concen_region_input.GeomName = "concen_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 smg.Geom.concen_region.Lower.X = 60.0
 smg.Geom.concen_region.Lower.Y = 80.0
@@ -117,227 +122,264 @@
 smg.Geom.concen_region.Upper.Y = 100.0
 smg.Geom.concen_region.Upper.Z = 6.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-smg.Geom.Perm.Names = 'background'
+smg.Geom.Perm.Names = "background"
 
-smg.Geom.background.Perm.Type = 'Constant'
+smg.Geom.background.Perm.Type = "Constant"
 smg.Geom.background.Perm.Value = 4.0
 
-smg.Perm.TensorType = 'TensorByGeom'
+smg.Perm.TensorType = "TensorByGeom"
 
-smg.Geom.Perm.TensorByGeom.Names = 'background'
+smg.Geom.Perm.TensorByGeom.Names = "background"
 
 smg.Geom.background.Perm.TensorValX = 1.0
 smg.Geom.background.Perm.TensorValY = 1.0
 smg.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-smg.SpecificStorage.Type = 'Constant'
-smg.SpecificStorage.GeomNames = 'domain'
+smg.SpecificStorage.Type = "Constant"
+smg.SpecificStorage.GeomNames = "domain"
 smg.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-smg.Phase.Names = 'water'
+smg.Phase.Names = "water"
 
-smg.Phase.water.Density.Type = 'Constant'
+smg.Phase.water.Density.Type = "Constant"
 smg.Phase.water.Density.Value = 1.0
 
-smg.Phase.water.Viscosity.Type = 'Constant'
+smg.Phase.water.Viscosity.Type = "Constant"
 smg.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-smg.Contaminants.Names = ''
+smg.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-smg.Geom.Retardation.GeomNames = ''
+smg.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 smg.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 smg.TimingInfo.BaseUnit = 1.0
 smg.TimingInfo.StartCount = 0
 smg.TimingInfo.StartTime = 0.0
 smg.TimingInfo.StopTime = 0.010
 smg.TimingInfo.DumpInterval = -1
-smg.TimeStep.Type = 'Constant'
+smg.TimeStep.Type = "Constant"
 smg.TimeStep.Value = 0.001
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-smg.Geom.Porosity.GeomNames = 'background'
+smg.Geom.Porosity.GeomNames = "background"
 
-smg.Geom.background.Porosity.Type = 'Constant'
+smg.Geom.background.Porosity.Type = "Constant"
 smg.Geom.background.Porosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-smg.Domain.GeomName = 'domain'
+smg.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-smg.Phase.RelPerm.Type = 'VanGenuchten'
-smg.Phase.RelPerm.GeomNames = 'domain'
+smg.Phase.RelPerm.Type = "VanGenuchten"
+smg.Phase.RelPerm.GeomNames = "domain"
 smg.Geom.domain.RelPerm.Alpha = 0.005
 smg.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-smg.Phase.Saturation.Type = 'VanGenuchten'
-smg.Phase.Saturation.GeomNames = 'domain'
+smg.Phase.Saturation.Type = "VanGenuchten"
+smg.Phase.Saturation.GeomNames = "domain"
 smg.Geom.domain.Saturation.Alpha = 0.005
 smg.Geom.domain.Saturation.N = 2.0
 smg.Geom.domain.Saturation.SRes = 0.2
 smg.Geom.domain.Saturation.SSat = 0.99
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-smg.Wells.Names = ''
+smg.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-smg.Cycle.Names = 'constant'
-smg.Cycle.constant.Names = 'alltime'
+smg.Cycle.Names = "constant"
+smg.Cycle.constant.Names = "alltime"
 smg.Cycle.constant.alltime.Length = 1
 smg.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-smg.BCPressure.PatchNames = 'left right front back bottom top'
+smg.BCPressure.PatchNames = "left right front back bottom top"
 
-smg.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-smg.Patch.left.BCPressure.Cycle = 'constant'
-smg.Patch.left.BCPressure.RefGeom = 'domain'
-smg.Patch.left.BCPressure.RefPatch = 'bottom'
+smg.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+smg.Patch.left.BCPressure.Cycle = "constant"
+smg.Patch.left.BCPressure.RefGeom = "domain"
+smg.Patch.left.BCPressure.RefPatch = "bottom"
 smg.Patch.left.BCPressure.alltime.Value = 5.0
 
-smg.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-smg.Patch.right.BCPressure.Cycle = 'constant'
-smg.Patch.right.BCPressure.RefGeom = 'domain'
-smg.Patch.right.BCPressure.RefPatch = 'bottom'
+smg.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+smg.Patch.right.BCPressure.Cycle = "constant"
+smg.Patch.right.BCPressure.RefGeom = "domain"
+smg.Patch.right.BCPressure.RefPatch = "bottom"
 smg.Patch.right.BCPressure.alltime.Value = 3.0
 
-smg.Patch.front.BCPressure.Type = 'FluxConst'
-smg.Patch.front.BCPressure.Cycle = 'constant'
+smg.Patch.front.BCPressure.Type = "FluxConst"
+smg.Patch.front.BCPressure.Cycle = "constant"
 smg.Patch.front.BCPressure.alltime.Value = 0.0
 
-smg.Patch.back.BCPressure.Type = 'FluxConst'
-smg.Patch.back.BCPressure.Cycle = 'constant'
+smg.Patch.back.BCPressure.Type = "FluxConst"
+smg.Patch.back.BCPressure.Cycle = "constant"
 smg.Patch.back.BCPressure.alltime.Value = 0.0
 
-smg.Patch.bottom.BCPressure.Type = 'FluxConst'
-smg.Patch.bottom.BCPressure.Cycle = 'constant'
+smg.Patch.bottom.BCPressure.Type = "FluxConst"
+smg.Patch.bottom.BCPressure.Cycle = "constant"
 smg.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-smg.Patch.top.BCPressure.Type = 'FluxConst'
-smg.Patch.top.BCPressure.Cycle = 'constant'
+smg.Patch.top.BCPressure.Type = "FluxConst"
+smg.Patch.top.BCPressure.Cycle = "constant"
 smg.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-smg.TopoSlopesX.Type = 'Constant'
-smg.TopoSlopesX.GeomNames = 'domain'
+smg.TopoSlopesX.Type = "Constant"
+smg.TopoSlopesX.GeomNames = "domain"
 smg.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-smg.TopoSlopesY.Type = 'Constant'
-smg.TopoSlopesY.GeomNames = 'domain'
+smg.TopoSlopesY.Type = "Constant"
+smg.TopoSlopesY.GeomNames = "domain"
 smg.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-smg.Mannings.Type = 'Constant'
-smg.Mannings.GeomNames = 'domain'
-smg.Mannings.Geom.domain.Value = 0.
+smg.Mannings.Type = "Constant"
+smg.Mannings.GeomNames = "domain"
+smg.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-smg.ICPressure.Type = 'HydroStaticPatch'
-smg.ICPressure.GeomNames = 'domain'
+smg.ICPressure.Type = "HydroStaticPatch"
+smg.ICPressure.GeomNames = "domain"
 smg.Geom.domain.ICPressure.Value = 3.0
-smg.Geom.domain.ICPressure.RefGeom = 'domain'
-smg.Geom.domain.ICPressure.RefPatch = 'bottom'
+smg.Geom.domain.ICPressure.RefGeom = "domain"
+smg.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-smg.PhaseSources.water.Type = 'Constant'
-smg.PhaseSources.water.GeomNames = 'background'
+smg.PhaseSources.water.Type = "Constant"
+smg.PhaseSources.water.GeomNames = "background"
 smg.PhaseSources.water.Geom.background.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-smg.KnownSolution = 'NoKnownSolution'
+smg.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-smg.Solver = 'Richards'
+smg.Solver = "Richards"
 smg.Solver.MaxIter = 5
 
 smg.Solver.Nonlinear.MaxIter = 10
 smg.Solver.Nonlinear.ResidualTol = 1e-9
-smg.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+smg.Solver.Nonlinear.EtaChoice = "EtaConstant"
 smg.Solver.Nonlinear.EtaValue = 1e-5
 smg.Solver.Nonlinear.UseJacobian = True
 smg.Solver.Nonlinear.DerivativeEpsilon = 1e-2
 
 smg.Solver.Linear.KrylovDimension = 10
 
-smg.Solver.Linear.Preconditioner = 'SMG'
+smg.Solver.Linear.Preconditioner = "SMG"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/smg')
-mkdir(dir_name)
-smg.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/smg")
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+smg.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+for i in range(6):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+rm(new_output_dir_name)
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/terrain_following_grid_overland.py b/test/python/terrain_following_grid_overland.py
index 90f1b8e38..689e96fe2 100644
--- a/test/python/terrain_following_grid_overland.py
+++ b/test/python/terrain_following_grid_overland.py
@@ -1,14 +1,17 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  This runs a simple 2D, terrain following problem with a 5% slope
 #  R. Maxwell 1-11
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
-tfgo = Run("terrain_following_grid_overland", __file__)
+run_name = "stormflow.terrain.dz5.5pc"
+tfgo = Run(run_name, __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 tfgo.FileVersion = 4
 
@@ -16,9 +19,9 @@
 tfgo.Process.Topology.Q = 1
 tfgo.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 tfgo.ComputationalGrid.Lower.X = 0.0
 tfgo.ComputationalGrid.Lower.Y = 0.0
@@ -30,20 +33,20 @@
 
 tfgo.ComputationalGrid.DX = 5.0
 tfgo.ComputationalGrid.DY = 1.0
-tfgo.ComputationalGrid.DZ = .05
+tfgo.ComputationalGrid.DZ = 0.05
 
-#---------------------------------------------------------
-# Domain Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Domain Geometry
+# ---------------------------------------------------------
 
-tfgo.GeomInput.Names = 'boxinput'
+tfgo.GeomInput.Names = "boxinput"
 
-tfgo.GeomInput.boxinput.InputType = 'Box'
-tfgo.GeomInput.boxinput.GeomName = 'domain'
+tfgo.GeomInput.boxinput.InputType = "Box"
+tfgo.GeomInput.boxinput.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 tfgo.Geom.domain.Lower.X = 0.0
 tfgo.Geom.domain.Lower.Y = 0.0
@@ -53,66 +56,66 @@
 tfgo.Geom.domain.Upper.Y = 1.0
 tfgo.Geom.domain.Upper.Z = 1.5
 
-tfgo.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+tfgo.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-tfgo.Geom.Perm.Names = 'domain'
+tfgo.Geom.Perm.Names = "domain"
 
-tfgo.Geom.domain.Perm.Type = 'Constant'
-tfgo.Geom.domain.Perm.Value = 10.
+tfgo.Geom.domain.Perm.Type = "Constant"
+tfgo.Geom.domain.Perm.Value = 10.0
 
-tfgo.Perm.TensorType = 'TensorByGeom'
+tfgo.Perm.TensorType = "TensorByGeom"
 
-tfgo.Geom.Perm.TensorByGeom.Names = 'domain'
+tfgo.Geom.Perm.TensorByGeom.Names = "domain"
 
 tfgo.Geom.domain.Perm.TensorValX = 1.0
 tfgo.Geom.domain.Perm.TensorValY = 1.0
 tfgo.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-tfgo.SpecificStorage.Type = 'Constant'
-tfgo.SpecificStorage.GeomNames = 'domain'
+tfgo.SpecificStorage.Type = "Constant"
+tfgo.SpecificStorage.GeomNames = "domain"
 tfgo.Geom.domain.SpecificStorage.Value = 1.0e-5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-tfgo.Phase.Names = 'water'
+tfgo.Phase.Names = "water"
 
-tfgo.Phase.water.Density.Type = 'Constant'
+tfgo.Phase.water.Density.Type = "Constant"
 tfgo.Phase.water.Density.Value = 1.0
 
-tfgo.Phase.water.Viscosity.Type = 'Constant'
+tfgo.Phase.water.Viscosity.Type = "Constant"
 tfgo.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-tfgo.Contaminants.Names = ''
+tfgo.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-tfgo.Geom.Retardation.GeomNames = ''
+tfgo.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 tfgo.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 # run for 2 hours @ 6min timesteps
 tfgo.TimingInfo.BaseUnit = 1.0
@@ -120,143 +123,143 @@
 tfgo.TimingInfo.StartTime = 0.0
 tfgo.TimingInfo.StopTime = 2.0
 tfgo.TimingInfo.DumpInterval = -1
-tfgo.TimeStep.Type = 'Constant'
+tfgo.TimeStep.Type = "Constant"
 tfgo.TimeStep.Value = 0.1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-tfgo.Geom.Porosity.GeomNames = 'domain'
-tfgo.Geom.domain.Porosity.Type = 'Constant'
+tfgo.Geom.Porosity.GeomNames = "domain"
+tfgo.Geom.domain.Porosity.Type = "Constant"
 tfgo.Geom.domain.Porosity.Value = 0.1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-tfgo.Domain.GeomName = 'domain'
+tfgo.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-tfgo.Phase.RelPerm.Type = 'VanGenuchten'
-tfgo.Phase.RelPerm.GeomNames = 'domain'
+tfgo.Phase.RelPerm.Type = "VanGenuchten"
+tfgo.Phase.RelPerm.GeomNames = "domain"
 
 tfgo.Geom.domain.RelPerm.Alpha = 6.0
-tfgo.Geom.domain.RelPerm.N = 2.
+tfgo.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-tfgo.Phase.Saturation.Type = 'VanGenuchten'
-tfgo.Phase.Saturation.GeomNames = 'domain'
+tfgo.Phase.Saturation.Type = "VanGenuchten"
+tfgo.Phase.Saturation.GeomNames = "domain"
 
 tfgo.Geom.domain.Saturation.Alpha = 6.0
-tfgo.Geom.domain.Saturation.N = 2.
+tfgo.Geom.domain.Saturation.N = 2.0
 tfgo.Geom.domain.Saturation.SRes = 0.2
 tfgo.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-tfgo.Wells.Names = ''
+tfgo.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-tfgo.Cycle.Names = 'constant rainrec'
-tfgo.Cycle.constant.Names = 'alltime'
+tfgo.Cycle.Names = "constant rainrec"
+tfgo.Cycle.constant.Names = "alltime"
 tfgo.Cycle.constant.alltime.Length = 1
 tfgo.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-tfgo.Cycle.rainrec.Names = 'rain rec'
+tfgo.Cycle.rainrec.Names = "rain rec"
 tfgo.Cycle.rainrec.rain.Length = 2
 tfgo.Cycle.rainrec.rec.Length = 2
 tfgo.Cycle.rainrec.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 tfgo.BCPressure.PatchNames = tfgo.Geom.domain.Patches
 
-tfgo.Patch.x_lower.BCPressure.Type = 'FluxConst'
-tfgo.Patch.x_lower.BCPressure.Cycle = 'constant'
+tfgo.Patch.x_lower.BCPressure.Type = "FluxConst"
+tfgo.Patch.x_lower.BCPressure.Cycle = "constant"
 tfgo.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-tfgo.Patch.y_lower.BCPressure.Type = 'FluxConst'
-tfgo.Patch.y_lower.BCPressure.Cycle = 'constant'
+tfgo.Patch.y_lower.BCPressure.Type = "FluxConst"
+tfgo.Patch.y_lower.BCPressure.Cycle = "constant"
 tfgo.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-tfgo.Patch.z_lower.BCPressure.Type = 'FluxConst'
-tfgo.Patch.z_lower.BCPressure.Cycle = 'constant'
+tfgo.Patch.z_lower.BCPressure.Type = "FluxConst"
+tfgo.Patch.z_lower.BCPressure.Cycle = "constant"
 tfgo.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-tfgo.Patch.x_upper.BCPressure.Type = 'FluxConst'
-tfgo.Patch.x_upper.BCPressure.Cycle = 'constant'
+tfgo.Patch.x_upper.BCPressure.Type = "FluxConst"
+tfgo.Patch.x_upper.BCPressure.Cycle = "constant"
 tfgo.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-tfgo.Patch.y_upper.BCPressure.Type = 'FluxConst'
-tfgo.Patch.y_upper.BCPressure.Cycle = 'constant'
+tfgo.Patch.y_upper.BCPressure.Type = "FluxConst"
+tfgo.Patch.y_upper.BCPressure.Cycle = "constant"
 tfgo.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-tfgo.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
+tfgo.Patch.z_upper.BCPressure.Type = "OverlandFlow"
 ##pfset Patch.z-upper.BCPressure.Type		      FluxConst
 
-tfgo.Patch.z_upper.BCPressure.Cycle = 'constant'
+tfgo.Patch.z_upper.BCPressure.Cycle = "constant"
 tfgo.Patch.z_upper.BCPressure.alltime.Value = 0.00
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-tfgo.TopoSlopesX.Type = 'Constant'
-tfgo.TopoSlopesX.GeomNames = 'domain'
+tfgo.TopoSlopesX.Type = "Constant"
+tfgo.TopoSlopesX.GeomNames = "domain"
 tfgo.TopoSlopesX.Geom.domain.Value = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-tfgo.TopoSlopesY.Type = 'Constant'
-tfgo.TopoSlopesY.GeomNames = 'domain'
+tfgo.TopoSlopesY.Type = "Constant"
+tfgo.TopoSlopesY.GeomNames = "domain"
 tfgo.TopoSlopesY.Geom.domain.Value = 0.00
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-tfgo.Mannings.Type = 'Constant'
-tfgo.Mannings.GeomNames = 'domain'
-tfgo.Mannings.Geom.domain.Value = 1.e-6
+tfgo.Mannings.Type = "Constant"
+tfgo.Mannings.GeomNames = "domain"
+tfgo.Mannings.Geom.domain.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-tfgo.PhaseSources.water.Type = 'Constant'
-tfgo.PhaseSources.water.GeomNames = 'domain'
+tfgo.PhaseSources.water.Type = "Constant"
+tfgo.PhaseSources.water.GeomNames = "domain"
 tfgo.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-tfgo.KnownSolution = 'NoKnownSolution'
+tfgo.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-tfgo.Solver = 'Richards'
+tfgo.Solver = "Richards"
 
 # setting this to True sets a subsurface slope that is the same as the topo slopes
 tfgo.Solver.TerrainFollowingGrid = True
@@ -265,40 +268,68 @@
 
 tfgo.Solver.Nonlinear.MaxIter = 300
 tfgo.Solver.Nonlinear.ResidualTol = 1e-6
-tfgo.Solver.Nonlinear.EtaChoice = 'Walker1'
+tfgo.Solver.Nonlinear.EtaChoice = "Walker1"
 tfgo.Solver.Nonlinear.EtaValue = 0.001
 tfgo.Solver.Nonlinear.UseJacobian = False
 tfgo.Solver.Nonlinear.DerivativeEpsilon = 1e-12
 tfgo.Solver.Nonlinear.StepTol = 1e-20
-tfgo.Solver.Nonlinear.Globalization = 'LineSearch'
+tfgo.Solver.Nonlinear.Globalization = "LineSearch"
 tfgo.Solver.Linear.KrylovDimension = 20
 tfgo.Solver.Linear.MaxRestart = 2
 
-tfgo.Solver.Linear.Preconditioner = 'MGSemi'
-tfgo.Solver.Linear.Preconditioner = 'PFMG'
-tfgo.Solver.Linear.Preconditioner.SymmetricMat = 'Symmetric'
+tfgo.Solver.Linear.Preconditioner = "MGSemi"
+tfgo.Solver.Linear.Preconditioner = "PFMG"
+tfgo.Solver.Linear.Preconditioner.SymmetricMat = "Symmetric"
 tfgo.Solver.Linear.Preconditioner.MGSemi.MaxIter = 1
 tfgo.Solver.Linear.Preconditioner.MGSemi.MaxLevels = 10
 tfgo.Solver.PrintSubsurf = False
-tfgo.Solver.Drop = 1E-20
-tfgo.Solver.AbsTol = 1E-12
+tfgo.Solver.Drop = 1e-20
+tfgo.Solver.AbsTol = 1e-12
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be 1m from the bottom of the domain, the top layer is initially dry
-tfgo.ICPressure.Type = 'HydroStaticPatch'
-tfgo.ICPressure.GeomNames = 'domain'
+tfgo.ICPressure.Type = "HydroStaticPatch"
+tfgo.ICPressure.GeomNames = "domain"
 tfgo.Geom.domain.ICPressure.Value = 1.0
 
-tfgo.Geom.domain.ICPressure.RefGeom = 'domain'
-tfgo.Geom.domain.ICPressure.RefPatch = 'z_lower'
+tfgo.Geom.domain.ICPressure.RefGeom = "domain"
+tfgo.Geom.domain.ICPressure.RefPatch = "z_lower"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/tfgo')
-mkdir(dir_name)
-tfgo.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/tfgo")
+mkdir(new_output_dir_name)
+correct_output_dir_name = get_absolute_path("../correct_output")
+tfgo.run(working_directory=new_output_dir_name)
+
+passed = True
+
+
+for i in range(21):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/tilted_v_with_reservoir_overland_flow.py b/test/python/tilted_v_with_reservoir_overland_flow.py
new file mode 100755
index 000000000..bfc66f8bd
--- /dev/null
+++ b/test/python/tilted_v_with_reservoir_overland_flow.py
@@ -0,0 +1,365 @@
+# -----------------------------------------------------------------------------
+# running different configuraitons of tilted V
+# -----------------------------------------------------------------------------
+import os
+
+import parflow
+from parflow import Run
+from parflow.tools.fs import mkdir, cp, get_absolute_path
+from parflow.tools.compare import pf_test_file_with_abs
+import sys
+import numpy as np
+
+run_name = "tilted_v_with_reservoir_overland_flow"
+overland = Run(run_name, __file__)
+
+# -----------------------------------------------------------------------------
+
+overland.FileVersion = 4
+
+overland.Process.Topology.P = 1
+overland.Process.Topology.Q = 1
+overland.Process.Topology.R = 1
+
+# ---------------------------------------------------------
+# Computational Grid
+# ---------------------------------------------------------
+gridnx = int(25)
+gridny = int(25)
+gridnz = int(1)
+griddx = 1.0
+griddy = 1.0
+griddz = 0.5
+
+overland.ComputationalGrid.Lower.X = 0.0
+overland.ComputationalGrid.Lower.Y = 0.0
+overland.ComputationalGrid.Lower.Z = 0.0
+
+overland.ComputationalGrid.NX = gridnx
+overland.ComputationalGrid.NY = gridny
+overland.ComputationalGrid.NZ = gridnz
+
+overland.ComputationalGrid.DX = griddx
+overland.ComputationalGrid.DY = griddy
+overland.ComputationalGrid.DZ = griddz
+
+# ---------------------------------------------------------
+# The Names of the GeomInputs
+# ---------------------------------------------------------
+overland.GeomInput.Names = "domaininput"
+
+overland.GeomInput.domaininput.GeomName = "domain"
+overland.GeomInput.domaininput.GeomNames = "domain"
+overland.GeomInput.domaininput.InputType = "SolidFile"
+overland.Geom.domain.Patches = "bottom side slope1 channel slope2"
+
+# -----------------------------------------------------------------------------
+# Perm
+# -----------------------------------------------------------------------------
+overland.Geom.Perm.Names = "domain"
+overland.Geom.domain.Perm.Type = "Constant"
+overland.Geom.domain.Perm.Value = 0.0000001
+
+overland.Perm.TensorType = "TensorByGeom"
+
+overland.Geom.Perm.TensorByGeom.Names = "domain"
+
+overland.Geom.domain.Perm.TensorValX = 1.0
+overland.Geom.domain.Perm.TensorValY = 1.0
+overland.Geom.domain.Perm.TensorValZ = 1.0
+
+# -----------------------------------------------------------------------------
+# Specific Storage
+# -----------------------------------------------------------------------------
+overland.SpecificStorage.Type = "Constant"
+overland.SpecificStorage.GeomNames = "domain"
+overland.Geom.domain.SpecificStorage.Value = 1.0e-4
+
+# -----------------------------------------------------------------------------
+# Phases
+# -----------------------------------------------------------------------------
+overland.Phase.Names = "water"
+
+overland.Phase.water.Density.Type = "Constant"
+overland.Phase.water.Density.Value = 1.0
+
+overland.Phase.water.Viscosity.Type = "Constant"
+overland.Phase.water.Viscosity.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Contaminants
+# -----------------------------------------------------------------------------
+overland.Contaminants.Names = ""
+
+# -----------------------------------------------------------------------------
+# Retardation
+# -----------------------------------------------------------------------------
+overland.Geom.Retardation.GeomNames = ""
+
+# -----------------------------------------------------------------------------
+# Gravity
+# -----------------------------------------------------------------------------
+overland.Gravity = 1.0
+
+# -----------------------------------------------------------------------------
+# Setup timing info
+# -----------------------------------------------------------------------------
+overland.TimingInfo.BaseUnit = 1
+overland.TimingInfo.StartCount = 0
+overland.TimingInfo.StartTime = 0.0
+overland.TimingInfo.StopTime = 11.0
+overland.TimingInfo.DumpInterval = 1.0
+overland.TimeStep.Type = "Constant"
+overland.TimeStep.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Porosity
+# -----------------------------------------------------------------------------
+
+overland.Geom.Porosity.GeomNames = "domain"
+overland.Geom.domain.Porosity.Type = "Constant"
+overland.Geom.domain.Porosity.Value = 0.01
+
+# -----------------------------------------------------------------------------
+# Domain
+# -----------------------------------------------------------------------------
+overland.Domain.GeomName = "domain"
+
+# -----------------------------------------------------------------------------
+# Relative Permeability
+# -----------------------------------------------------------------------------
+overland.Phase.RelPerm.Type = "VanGenuchten"
+overland.Phase.RelPerm.GeomNames = "domain"
+
+overland.Geom.domain.RelPerm.Alpha = 6.0
+overland.Geom.domain.RelPerm.N = 2.0
+
+# ---------------------------------------------------------
+# Saturation
+# ---------------------------------------------------------
+overland.Phase.Saturation.Type = "VanGenuchten"
+overland.Phase.Saturation.GeomNames = "domain"
+
+overland.Geom.domain.Saturation.Alpha = 6.0
+overland.Geom.domain.Saturation.N = 2.0
+overland.Geom.domain.Saturation.SRes = 0.2
+overland.Geom.domain.Saturation.SSat = 1.0
+
+# -----------------------------------------------------------------------------
+# Wells
+# -----------------------------------------------------------------------------
+overland.Wells.Names = ""
+
+# -----------------------------------------------------------------------------
+# Time Cycles
+# -----------------------------------------------------------------------------
+overland.Cycle.Names = "constant rainrec"
+overland.Cycle.constant.Names = "alltime"
+overland.Cycle.constant.alltime.Length = 1
+overland.Cycle.constant.Repeat = -1
+
+overland.Cycle.rainrec.Names = "rain"
+overland.Cycle.rainrec.rain.Length = 1
+overland.Cycle.rainrec.Repeat = -1
+
+# -----------------------------------------------------------------------------
+# Boundary Conditions: Pressure
+# -----------------------------------------------------------------------------
+
+overland.BCPressure.PatchNames = overland.Geom.domain.Patches
+
+overland.Patch.side.BCPressure.Type = "FluxConst"
+overland.Patch.side.BCPressure.Cycle = "constant"
+overland.Patch.side.BCPressure.alltime.Value = 0.0
+
+overland.Patch.bottom.BCPressure.Type = "FluxConst"
+overland.Patch.bottom.BCPressure.Cycle = "constant"
+overland.Patch.bottom.BCPressure.alltime.Value = 0.0
+
+overland.Patch.slope1.BCPressure.Cycle = "constant"
+overland.Patch.slope1.BCPressure.alltime.Value = -0.001
+
+overland.Patch.slope2.BCPressure.Cycle = "constant"
+overland.Patch.slope2.BCPressure.alltime.Value = -0.001
+
+overland.Patch.channel.BCPressure.Cycle = "constant"
+overland.Patch.channel.BCPressure.alltime.Value = -0.001
+
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
+overland.Mannings.Type = "Constant"
+overland.Mannings.GeomNames = "domain"
+overland.Mannings.Geom.domain.Value = 3.0e-6
+
+# -----------------------------------------------------------------------------
+# Phase sources:
+# -----------------------------------------------------------------------------
+overland.PhaseSources.water.Type = "Constant"
+overland.PhaseSources.water.GeomNames = "domain"
+overland.PhaseSources.water.Geom.domain.Value = 0.0
+
+# -----------------------------------------------------------------------------
+# Exact solution specification for error calculations
+# -----------------------------------------------------------------------------
+overland.KnownSolution = "NoKnownSolution"
+
+# -----------------------------------------------------------------------------
+# Slopes
+# -----------------------------------------------------------------------------
+overland.TopoSlopesX.Type = "PFBFile"
+overland.TopoSlopesX.GeomNames = "domain"
+overland.TopoSlopesX.FileName = "slopex.pfb"
+
+overland.TopoSlopesY.Type = "PFBFile"
+overland.TopoSlopesY.GeomNames = "domain"
+overland.TopoSlopesY.FileName = "slopey.pfb"
+
+# -----------------------------------------------------------------------------
+# Set solver parameters
+# -----------------------------------------------------------------------------
+overland.Solver = "Richards"
+overland.Solver.MaxIter = 2500
+
+overland.Solver.Nonlinear.MaxIter = 100
+overland.Solver.Nonlinear.ResidualTol = 1e-9
+overland.Solver.Nonlinear.EtaChoice = "EtaConstant"
+overland.Solver.Nonlinear.EtaValue = 0.01
+overland.Solver.Nonlinear.UseJacobian = False
+overland.Solver.Nonlinear.DerivativeEpsilon = 1e-15
+overland.Solver.Nonlinear.StepTol = 1e-20
+overland.Solver.Nonlinear.Globalization = "LineSearch"
+overland.Solver.Linear.KrylovDimension = 50
+overland.Solver.Linear.MaxRestart = 2
+
+
+overland.Solver.Linear.Preconditioner = "PFMG"
+overland.Solver.PrintSubsurf = True
+overland.Solver.PrintSlopes = True
+overland.Solver.PrintMannings = True
+overland.Solver.Drop = 1e-20
+overland.Solver.AbsTol = 1e-10
+
+overland.Solver.WriteSiloSubsurfData = False
+overland.Solver.WriteSiloPressure = False
+overland.Solver.WriteSiloSlopes = False
+
+overland.Solver.WriteSiloSaturation = False
+overland.Solver.WriteSiloConcentration = False
+
+
+# ---------------------------------------------------------
+# Initial conditions: water pressure
+# ---------------------------------------------------------
+# set water table to be at the bottom of the domain, the top layer is initially dry
+overland.ICPressure.Type = "PFBFile"
+overland.ICPressure.GeomNames = "domain"
+overland.Geom.domain.ICPressure.FileName = (
+    "tilted_v_with_reservoir_initial_pressure.pfb"
+)
+
+overland.Geom.domain.ICPressure.RefGeom = "domain"
+overland.Geom.domain.ICPressure.RefPatch = "bottom"
+
+
+# -----------------------------------------------------------------------------
+# defining write function to save some space in the loop
+# -----------------------------------------------------------------------------
+def write_pfb_to_run_dir(myarray, fout, run_dir):
+    fout = get_absolute_path(os.path.join(run_dir, fout))
+    parflow.write_pfb(fout, myarray)
+
+
+def dist_and_run(run_dir):
+    overland.dist(os.path.join(run_dir, "slopex.pfb"))
+    overland.dist(os.path.join(run_dir, "slopey.pfb"))
+
+    overland.write(working_directory=run_dir)
+    overland.write(file_format="yaml", working_directory=run_dir)
+    overland.write(file_format="json", working_directory=run_dir)
+    overland.run(working_directory=run_dir)
+
+
+# Make an x direction solid file
+inputs_dir = get_absolute_path("../../test/input")
+solid_name = f"{inputs_dir}/tilted_v_with_reservoir"
+solid_fname = solid_name + ".pfsol"
+
+# -----------------------------------------------------------------------------
+# setup the overland patch types
+# -----------------------------------------------------------------------------
+overland.Patch.slope1.BCPressure.Type = "OverlandFlow"
+overland.Patch.slope2.BCPressure.Type = "OverlandFlow"
+overland.Patch.channel.BCPressure.Type = "OverlandFlow"
+
+# -----------------------------------------------------------------------------
+# Draining right-- (negative X slope channel)
+# -----------------------------------------------------------------------------
+correct_output_dir = get_absolute_path("../correct_output")
+print(correct_output_dir)
+
+
+run_dir = get_absolute_path("test_output/tilted_v_with_reservoir_overland_flow")
+
+mkdir(run_dir)
+
+cp(solid_fname, run_dir)
+cp(f"{inputs_dir}/tilted_v_with_reservoir_initial_pressure.pfb", run_dir)
+cp(f"{inputs_dir}/tilted_v_with_reservoir_initial_pressure.pfb.dist", run_dir)
+overland.GeomInput.domaininput.FileName = solid_fname
+
+# Make slope files
+MIDDLE_ROW = int(np.floor(gridny / 2))
+slopey = np.full((gridnz, gridny, gridnx), 1.0)
+slopey[:, 0:MIDDLE_ROW, :] = np.round(-0.01, 2)
+slopey[:, MIDDLE_ROW + 1 :, :] = np.round(0.01, 2)
+slopey[:, MIDDLE_ROW, :] = np.round(0.00, 2)
+write_pfb_to_run_dir(slopey, "slopey.pfb", run_dir)
+
+slopex = np.ones((gridnz, gridny, gridnx))
+slopex = np.round(slopex * -0.01, 2)
+write_pfb_to_run_dir(slopex, "slopex.pfb", run_dir)
+
+overland.Reservoirs.Names = "reservoir"
+overland.Reservoirs.Overland_Flow_Solver = "OverlandFlow"
+overland.Reservoirs.reservoir.Intake_X = 12.5
+overland.Reservoirs.reservoir.Intake_Y = 12.5
+overland.Reservoirs.reservoir.Release_X = 13.5
+overland.Reservoirs.reservoir.Release_Y = 12.5
+overland.Reservoirs.reservoir.Has_Secondary_Intake_Cell = 0
+overland.Reservoirs.reservoir.Secondary_Intake_X = -1
+overland.Reservoirs.reservoir.Secondary_Intake_Y = -1
+
+overland.Reservoirs.reservoir.Max_Storage = 100
+overland.Reservoirs.reservoir.Storage = 50
+overland.Reservoirs.reservoir.Min_Release_Storage = 0
+overland.Reservoirs.reservoir.Release_Rate = 0
+
+dist_and_run(run_dir)
+
+passed = True
+
+i = 10
+timestep = str(i).rjust(5, "0")
+
+sig_digits = 8
+abs_value = 1e-12
+
+test_files = ["press"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        run_dir + filename,
+        correct_output_dir + filename,
+        f"Max difference in {filename}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/tilted_v_with_reservoir_overland_kinematic.py b/test/python/tilted_v_with_reservoir_overland_kinematic.py
new file mode 100755
index 000000000..09a059803
--- /dev/null
+++ b/test/python/tilted_v_with_reservoir_overland_kinematic.py
@@ -0,0 +1,365 @@
+# -----------------------------------------------------------------------------
+# running different configuraitons of tilted V
+# -----------------------------------------------------------------------------
+import os
+
+import parflow
+from parflow import Run
+from parflow.tools.fs import mkdir, cp, get_absolute_path
+from parflow.tools.compare import pf_test_file_with_abs
+import sys
+import numpy as np
+
+run_name = "tilted_v_with_reservoir_overland_kinematic"
+overland = Run(run_name, __file__)
+
+# -----------------------------------------------------------------------------
+
+overland.FileVersion = 4
+
+overland.Process.Topology.P = 1
+overland.Process.Topology.Q = 1
+overland.Process.Topology.R = 1
+
+# ---------------------------------------------------------
+# Computational Grid
+# ---------------------------------------------------------
+gridnx = int(25)
+gridny = int(25)
+gridnz = int(1)
+griddx = 1.0
+griddy = 1.0
+griddz = 0.5
+
+overland.ComputationalGrid.Lower.X = 0.0
+overland.ComputationalGrid.Lower.Y = 0.0
+overland.ComputationalGrid.Lower.Z = 0.0
+
+overland.ComputationalGrid.NX = gridnx
+overland.ComputationalGrid.NY = gridny
+overland.ComputationalGrid.NZ = gridnz
+
+overland.ComputationalGrid.DX = griddx
+overland.ComputationalGrid.DY = griddy
+overland.ComputationalGrid.DZ = griddz
+
+# ---------------------------------------------------------
+# The Names of the GeomInputs
+# ---------------------------------------------------------
+overland.GeomInput.Names = "domaininput"
+
+overland.GeomInput.domaininput.GeomName = "domain"
+overland.GeomInput.domaininput.GeomNames = "domain"
+overland.GeomInput.domaininput.InputType = "SolidFile"
+overland.Geom.domain.Patches = "bottom side slope1 channel slope2"
+
+# -----------------------------------------------------------------------------
+# Perm
+# -----------------------------------------------------------------------------
+overland.Geom.Perm.Names = "domain"
+overland.Geom.domain.Perm.Type = "Constant"
+overland.Geom.domain.Perm.Value = 0.0000001
+
+overland.Perm.TensorType = "TensorByGeom"
+
+overland.Geom.Perm.TensorByGeom.Names = "domain"
+
+overland.Geom.domain.Perm.TensorValX = 1.0
+overland.Geom.domain.Perm.TensorValY = 1.0
+overland.Geom.domain.Perm.TensorValZ = 1.0
+
+# -----------------------------------------------------------------------------
+# Specific Storage
+# -----------------------------------------------------------------------------
+overland.SpecificStorage.Type = "Constant"
+overland.SpecificStorage.GeomNames = "domain"
+overland.Geom.domain.SpecificStorage.Value = 1.0e-4
+
+# -----------------------------------------------------------------------------
+# Phases
+# -----------------------------------------------------------------------------
+overland.Phase.Names = "water"
+
+overland.Phase.water.Density.Type = "Constant"
+overland.Phase.water.Density.Value = 1.0
+
+overland.Phase.water.Viscosity.Type = "Constant"
+overland.Phase.water.Viscosity.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Contaminants
+# -----------------------------------------------------------------------------
+overland.Contaminants.Names = ""
+
+# -----------------------------------------------------------------------------
+# Retardation
+# -----------------------------------------------------------------------------
+overland.Geom.Retardation.GeomNames = ""
+
+# -----------------------------------------------------------------------------
+# Gravity
+# -----------------------------------------------------------------------------
+overland.Gravity = 1.0
+
+# -----------------------------------------------------------------------------
+# Setup timing info
+# -----------------------------------------------------------------------------
+overland.TimingInfo.BaseUnit = 1
+overland.TimingInfo.StartCount = 0
+overland.TimingInfo.StartTime = 0.0
+overland.TimingInfo.StopTime = 11.0
+overland.TimingInfo.DumpInterval = 1.0
+overland.TimeStep.Type = "Constant"
+overland.TimeStep.Value = 1.0
+
+# -----------------------------------------------------------------------------
+# Porosity
+# -----------------------------------------------------------------------------
+
+overland.Geom.Porosity.GeomNames = "domain"
+overland.Geom.domain.Porosity.Type = "Constant"
+overland.Geom.domain.Porosity.Value = 0.01
+
+# -----------------------------------------------------------------------------
+# Domain
+# -----------------------------------------------------------------------------
+overland.Domain.GeomName = "domain"
+
+# -----------------------------------------------------------------------------
+# Relative Permeability
+# -----------------------------------------------------------------------------
+overland.Phase.RelPerm.Type = "VanGenuchten"
+overland.Phase.RelPerm.GeomNames = "domain"
+
+overland.Geom.domain.RelPerm.Alpha = 6.0
+overland.Geom.domain.RelPerm.N = 2.0
+
+# ---------------------------------------------------------
+# Saturation
+# ---------------------------------------------------------
+overland.Phase.Saturation.Type = "VanGenuchten"
+overland.Phase.Saturation.GeomNames = "domain"
+
+overland.Geom.domain.Saturation.Alpha = 6.0
+overland.Geom.domain.Saturation.N = 2.0
+overland.Geom.domain.Saturation.SRes = 0.2
+overland.Geom.domain.Saturation.SSat = 1.0
+
+# -----------------------------------------------------------------------------
+# Wells
+# -----------------------------------------------------------------------------
+overland.Wells.Names = ""
+
+# -----------------------------------------------------------------------------
+# Time Cycles
+# -----------------------------------------------------------------------------
+overland.Cycle.Names = "constant rainrec"
+overland.Cycle.constant.Names = "alltime"
+overland.Cycle.constant.alltime.Length = 1
+overland.Cycle.constant.Repeat = -1
+
+overland.Cycle.rainrec.Names = "rain"
+overland.Cycle.rainrec.rain.Length = 1
+overland.Cycle.rainrec.Repeat = -1
+
+# -----------------------------------------------------------------------------
+# Boundary Conditions: Pressure
+# -----------------------------------------------------------------------------
+
+overland.BCPressure.PatchNames = overland.Geom.domain.Patches
+
+overland.Patch.side.BCPressure.Type = "FluxConst"
+overland.Patch.side.BCPressure.Cycle = "constant"
+overland.Patch.side.BCPressure.alltime.Value = 0.0
+
+overland.Patch.bottom.BCPressure.Type = "FluxConst"
+overland.Patch.bottom.BCPressure.Cycle = "constant"
+overland.Patch.bottom.BCPressure.alltime.Value = 0.0
+
+overland.Patch.slope1.BCPressure.Cycle = "constant"
+overland.Patch.slope1.BCPressure.alltime.Value = -0.001
+
+overland.Patch.slope2.BCPressure.Cycle = "constant"
+overland.Patch.slope2.BCPressure.alltime.Value = -0.001
+
+overland.Patch.channel.BCPressure.Cycle = "constant"
+overland.Patch.channel.BCPressure.alltime.Value = -0.001
+
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
+overland.Mannings.Type = "Constant"
+overland.Mannings.GeomNames = "domain"
+overland.Mannings.Geom.domain.Value = 3.0e-6
+
+# -----------------------------------------------------------------------------
+# Phase sources:
+# -----------------------------------------------------------------------------
+overland.PhaseSources.water.Type = "Constant"
+overland.PhaseSources.water.GeomNames = "domain"
+overland.PhaseSources.water.Geom.domain.Value = 0.0
+
+# -----------------------------------------------------------------------------
+# Exact solution specification for error calculations
+# -----------------------------------------------------------------------------
+overland.KnownSolution = "NoKnownSolution"
+
+# -----------------------------------------------------------------------------
+# Slopes
+# -----------------------------------------------------------------------------
+overland.TopoSlopesX.Type = "PFBFile"
+overland.TopoSlopesX.GeomNames = "domain"
+overland.TopoSlopesX.FileName = "slopex.pfb"
+
+overland.TopoSlopesY.Type = "PFBFile"
+overland.TopoSlopesY.GeomNames = "domain"
+overland.TopoSlopesY.FileName = "slopey.pfb"
+
+# -----------------------------------------------------------------------------
+# Set solver parameters
+# -----------------------------------------------------------------------------
+overland.Solver = "Richards"
+overland.Solver.MaxIter = 2500
+
+overland.Solver.Nonlinear.MaxIter = 100
+overland.Solver.Nonlinear.ResidualTol = 1e-9
+overland.Solver.Nonlinear.EtaChoice = "EtaConstant"
+overland.Solver.Nonlinear.EtaValue = 0.01
+overland.Solver.Nonlinear.UseJacobian = False
+overland.Solver.Nonlinear.DerivativeEpsilon = 1e-15
+overland.Solver.Nonlinear.StepTol = 1e-20
+overland.Solver.Nonlinear.Globalization = "LineSearch"
+overland.Solver.Linear.KrylovDimension = 50
+overland.Solver.Linear.MaxRestart = 2
+overland.Solver.OverlandKinematic.Epsilon = 1e-5
+
+overland.Solver.Linear.Preconditioner = "PFMG"
+overland.Solver.PrintSubsurf = True
+overland.Solver.PrintSlopes = True
+overland.Solver.PrintMannings = True
+overland.Solver.Drop = 1e-20
+overland.Solver.AbsTol = 1e-10
+
+overland.Solver.WriteSiloSubsurfData = False
+overland.Solver.WriteSiloPressure = False
+overland.Solver.WriteSiloSlopes = False
+
+overland.Solver.WriteSiloSaturation = False
+overland.Solver.WriteSiloConcentration = False
+
+
+# ---------------------------------------------------------
+# Initial conditions: water pressure
+# ---------------------------------------------------------
+# set water table to be at the bottom of the domain, the top layer is initially dry
+overland.ICPressure.Type = "PFBFile"
+overland.ICPressure.GeomNames = "domain"
+overland.Geom.domain.ICPressure.FileName = (
+    "tilted_v_with_reservoir_initial_pressure.pfb"
+)
+
+overland.Geom.domain.ICPressure.RefGeom = "domain"
+overland.Geom.domain.ICPressure.RefPatch = "bottom"
+
+
+# -----------------------------------------------------------------------------
+# defining write function to save some space in the loop
+# -----------------------------------------------------------------------------
+def write_pfb_to_run_dir(myarray, fout, run_dir):
+    fout = get_absolute_path(os.path.join(run_dir, fout))
+    parflow.write_pfb(fout, myarray)
+
+
+def dist_and_run(run_dir):
+    overland.dist(os.path.join(run_dir, "slopex.pfb"))
+    overland.dist(os.path.join(run_dir, "slopey.pfb"))
+
+    overland.write(working_directory=run_dir)
+    overland.write(file_format="yaml", working_directory=run_dir)
+    overland.write(file_format="json", working_directory=run_dir)
+    overland.run(working_directory=run_dir)
+
+
+# Make an x direction solid file
+inputs_dir = get_absolute_path("../../test/input")
+solid_name = f"{inputs_dir}/tilted_v_with_reservoir"
+solid_fname = solid_name + ".pfsol"
+
+# -----------------------------------------------------------------------------
+# setup the overland patch types
+# -----------------------------------------------------------------------------
+overland.Patch.slope1.BCPressure.Type = "OverlandKinematic"
+overland.Patch.slope2.BCPressure.Type = "OverlandKinematic"
+overland.Patch.channel.BCPressure.Type = "OverlandKinematic"
+
+# -----------------------------------------------------------------------------
+# Draining right-- (negative X slope channel)
+# -----------------------------------------------------------------------------
+correct_output_dir = get_absolute_path("../correct_output")
+print(correct_output_dir)
+
+
+run_dir = get_absolute_path("test_output/tilted_v_with_reservoir_overland_kinematic")
+
+mkdir(run_dir)
+
+cp(solid_fname, run_dir)
+cp(f"{inputs_dir}/tilted_v_with_reservoir_initial_pressure.pfb", run_dir)
+cp(f"{inputs_dir}/tilted_v_with_reservoir_initial_pressure.pfb.dist", run_dir)
+overland.GeomInput.domaininput.FileName = solid_fname
+
+# Make slope files
+MIDDLE_ROW = int(np.floor(gridny / 2))
+slopey = np.full((gridnz, gridny, gridnx), 1.0)
+slopey[:, 0:MIDDLE_ROW, :] = np.round(-0.01, 2)
+slopey[:, MIDDLE_ROW + 1 :, :] = np.round(0.01, 2)
+# slopey[:, MIDDLE_ROW, :] = np.round(0.00,2)
+write_pfb_to_run_dir(slopey, "slopey.pfb", run_dir)
+
+slopex = np.ones((gridnz, gridny, gridnx))
+slopex = np.round(slopex * -0.01, 2)
+write_pfb_to_run_dir(slopex, "slopex.pfb", run_dir)
+
+overland.Reservoirs.Names = "reservoir"
+overland.Reservoirs.Overland_Flow_Solver = "OverlandKinematic"
+overland.Reservoirs.reservoir.Intake_X = 12.5
+overland.Reservoirs.reservoir.Intake_Y = 12.5
+overland.Reservoirs.reservoir.Release_X = 13.5
+overland.Reservoirs.reservoir.Release_Y = 12.5
+overland.Reservoirs.reservoir.Has_Secondary_Intake_Cell = 0
+overland.Reservoirs.reservoir.Secondary_Intake_X = -1
+overland.Reservoirs.reservoir.Secondary_Intake_Y = -1
+
+overland.Reservoirs.reservoir.Max_Storage = 100
+overland.Reservoirs.reservoir.Storage = 50
+overland.Reservoirs.reservoir.Min_Release_Storage = 0
+overland.Reservoirs.reservoir.Release_Rate = 0
+
+dist_and_run(run_dir)
+
+passed = True
+
+i = 10
+timestep = str(i).rjust(5, "0")
+
+sig_digits = 4
+abs_value = 1e-12
+
+test_files = ["press"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.{timestep}.pfb"
+    if not pf_test_file_with_abs(
+        run_dir + filename,
+        correct_output_dir + filename,
+        f"Max difference in {filename}",
+        abs_value,
+        sig_digits,
+    ):
+        passed = False
+
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/van-genuchten-file.py b/test/python/van-genuchten-file.py
index b41466efb..cf22557ce 100644
--- a/test/python/van-genuchten-file.py
+++ b/test/python/van-genuchten-file.py
@@ -1,32 +1,35 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  This runs the basic default_richards test case.
 #  This run, as written in this input file, should take
 #  3 nonlinear iterations.
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import cp, mkdir, get_absolute_path, chdir
+from parflow.tools.compare import pf_test_file
 
-vgf = Run("van-genuchten-file", __file__)
+run_name = "default_richards"
+vgf = Run(run_name, __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Creating and navigating to output directory
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/vgf')
-mkdir(dir_name)
-chdir(dir_name)
+new_output_dir_name = get_absolute_path("test_output/van-genuchten-file")
+mkdir(new_output_dir_name)
+correct_output_dir_name = get_absolute_path("../correct_output")
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Copying parameter files
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-cp('$PF_SRC/test/input/van-genuchten-alpha.pfb')
-cp('$PF_SRC/test/input/van-genuchten-n.pfb')
-cp('$PF_SRC/test/input/van-genuchten-sr.pfb')
-cp('$PF_SRC/test/input/van-genuchten-ssat.pfb')
+cp("$PF_SRC/test/input/van-genuchten-alpha.pfb", new_output_dir_name)
+cp("$PF_SRC/test/input/van-genuchten-n.pfb", new_output_dir_name)
+cp("$PF_SRC/test/input/van-genuchten-sr.pfb", new_output_dir_name)
+cp("$PF_SRC/test/input/van-genuchten-ssat.pfb", new_output_dir_name)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 vgf.FileVersion = 4
 
@@ -34,9 +37,9 @@
 vgf.Process.Topology.Q = 1
 vgf.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 vgf.ComputationalGrid.Lower.X = -10.0
 vgf.ComputationalGrid.Lower.Y = 10.0
@@ -46,26 +49,28 @@
 vgf.ComputationalGrid.DY = 10.666666666666666
 vgf.ComputationalGrid.DZ = 1.0
 
-vgf.ComputationalGrid.NX = 10
-vgf.ComputationalGrid.NY = 10
+vgf.ComputationalGrid.NX = 18
+vgf.ComputationalGrid.NY = 15
 vgf.ComputationalGrid.NZ = 8
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-vgf.GeomInput.Names = 'domain_input background_input source_region_input concen_region_input'
+vgf.GeomInput.Names = (
+    "domain_input background_input source_region_input concen_region_input"
+)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-vgf.GeomInput.domain_input.InputType = 'Box'
-vgf.GeomInput.domain_input.GeomName = 'domain'
+vgf.GeomInput.domain_input.InputType = "Box"
+vgf.GeomInput.domain_input.GeomName = "domain"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Domain Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 vgf.Geom.domain.Lower.X = -10.0
 vgf.Geom.domain.Lower.Y = 10.0
@@ -75,18 +80,18 @@
 vgf.Geom.domain.Upper.Y = 170.0
 vgf.Geom.domain.Upper.Z = 9.0
 
-vgf.Geom.domain.Patches = 'left right front back bottom top'
+vgf.Geom.domain.Patches = "left right front back bottom top"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-vgf.GeomInput.background_input.InputType = 'Box'
-vgf.GeomInput.background_input.GeomName = 'background'
+vgf.GeomInput.background_input.InputType = "Box"
+vgf.GeomInput.background_input.GeomName = "background"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Background Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 vgf.Geom.background.Lower.X = -99999999.0
 vgf.Geom.background.Lower.Y = -99999999.0
@@ -96,16 +101,16 @@
 vgf.Geom.background.Upper.Y = 99999999.0
 vgf.Geom.background.Upper.Z = 99999999.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-vgf.GeomInput.source_region_input.InputType = 'Box'
-vgf.GeomInput.source_region_input.GeomName = 'source_region'
+vgf.GeomInput.source_region_input.InputType = "Box"
+vgf.GeomInput.source_region_input.GeomName = "source_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Source_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 vgf.Geom.source_region.Lower.X = 65.56
 vgf.Geom.source_region.Lower.Y = 79.34
@@ -115,16 +120,16 @@
 vgf.Geom.source_region.Upper.Y = 89.99
 vgf.Geom.source_region.Upper.Z = 5.5
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-vgf.GeomInput.concen_region_input.InputType = 'Box'
-vgf.GeomInput.concen_region_input.GeomName = 'concen_region'
+vgf.GeomInput.concen_region_input.InputType = "Box"
+vgf.GeomInput.concen_region_input.GeomName = "concen_region"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Concen_Region Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 vgf.Geom.concen_region.Lower.X = 60.0
 vgf.Geom.concen_region.Lower.Y = 80.0
@@ -134,105 +139,105 @@
 vgf.Geom.concen_region.Upper.Y = 100.0
 vgf.Geom.concen_region.Upper.Z = 6.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vgf.Geom.Perm.Names = 'background'
+vgf.Geom.Perm.Names = "background"
 
-vgf.Geom.background.Perm.Type = 'Constant'
+vgf.Geom.background.Perm.Type = "Constant"
 vgf.Geom.background.Perm.Value = 4.0
 
-vgf.Perm.TensorType = 'TensorByGeom'
+vgf.Perm.TensorType = "TensorByGeom"
 
-vgf.Geom.Perm.TensorByGeom.Names = 'background'
+vgf.Geom.Perm.TensorByGeom.Names = "background"
 
 vgf.Geom.background.Perm.TensorValX = 1.0
 vgf.Geom.background.Perm.TensorValY = 1.0
 vgf.Geom.background.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vgf.SpecificStorage.Type = 'Constant'
-vgf.SpecificStorage.GeomNames = 'domain'
+vgf.SpecificStorage.Type = "Constant"
+vgf.SpecificStorage.GeomNames = "domain"
 vgf.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vgf.Phase.Names = 'water'
+vgf.Phase.Names = "water"
 
-vgf.Phase.water.Density.Type = 'Constant'
+vgf.Phase.water.Density.Type = "Constant"
 vgf.Phase.water.Density.Value = 1.0
 
-vgf.Phase.water.Viscosity.Type = 'Constant'
+vgf.Phase.water.Viscosity.Type = "Constant"
 vgf.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vgf.Contaminants.Names = ''
+vgf.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vgf.Geom.Retardation.GeomNames = ''
+vgf.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 vgf.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 vgf.TimingInfo.BaseUnit = 1.0
 vgf.TimingInfo.StartCount = 0
 vgf.TimingInfo.StartTime = 0.0
 vgf.TimingInfo.StopTime = 0.010
 vgf.TimingInfo.DumpInterval = -1
-vgf.TimeStep.Type = 'Constant'
+vgf.TimeStep.Type = "Constant"
 vgf.TimeStep.Value = 0.001
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vgf.Geom.Porosity.GeomNames = 'background'
+vgf.Geom.Porosity.GeomNames = "background"
 
-vgf.Geom.background.Porosity.Type = 'Constant'
+vgf.Geom.background.Porosity.Type = "Constant"
 vgf.Geom.background.Porosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vgf.Domain.GeomName = 'domain'
+vgf.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vgf.Phase.RelPerm.Type = 'VanGenuchten'
-vgf.Phase.RelPerm.GeomNames = 'domain'
+vgf.Phase.RelPerm.Type = "VanGenuchten"
+vgf.Phase.RelPerm.GeomNames = "domain"
 
 vgf.Phase.RelPerm.VanGenuchten.File = 1
-vgf.Geom.domain.RelPerm.Alpha.Filename = 'van-genuchten-alpha.pfb'
-vgf.Geom.domain.RelPerm.N.Filename = 'van-genuchten-n.pfb'
+vgf.Geom.domain.RelPerm.Alpha.Filename = "van-genuchten-alpha.pfb"
+vgf.Geom.domain.RelPerm.N.Filename = "van-genuchten-n.pfb"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-vgf.Phase.Saturation.Type = 'VanGenuchten'
-vgf.Phase.Saturation.GeomNames = 'domain'
+vgf.Phase.Saturation.Type = "VanGenuchten"
+vgf.Phase.Saturation.GeomNames = "domain"
 
 vgf.Geom.domain.Saturation.Alpha = 0.005
 vgf.Geom.domain.Saturation.N = 2.0
@@ -240,134 +245,170 @@
 vgf.Geom.domain.Saturation.SSat = 0.99
 
 vgf.Phase.Saturation.VanGenuchten.File = 1
-vgf.Geom.domain.Saturation.Alpha.Filename = 'van-genuchten-alpha.pfb'
-vgf.Geom.domain.Saturation.N.Filename = 'van-genuchten-n.pfb'
-vgf.Geom.domain.Saturation.SRes.Filename = 'van-genuchten-sr.pfb'
-vgf.Geom.domain.Saturation.SSat.Filename = 'van-genuchten-ssat.pfb'
+vgf.Geom.domain.Saturation.Alpha.Filename = "van-genuchten-alpha.pfb"
+vgf.Geom.domain.Saturation.N.Filename = "van-genuchten-n.pfb"
+vgf.Geom.domain.Saturation.SRes.Filename = "van-genuchten-sr.pfb"
+vgf.Geom.domain.Saturation.SSat.Filename = "van-genuchten-ssat.pfb"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vgf.Wells.Names = ''
+vgf.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vgf.Cycle.Names = 'constant'
-vgf.Cycle.constant.Names = 'alltime'
+vgf.Cycle.Names = "constant"
+vgf.Cycle.constant.Names = "alltime"
 vgf.Cycle.constant.alltime.Length = 1
 vgf.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vgf.BCPressure.PatchNames = 'left right front back bottom top'
+vgf.BCPressure.PatchNames = "left right front back bottom top"
 
-vgf.Patch.left.BCPressure.Type = 'DirEquilRefPatch'
-vgf.Patch.left.BCPressure.Cycle = 'constant'
-vgf.Patch.left.BCPressure.RefGeom = 'domain'
-vgf.Patch.left.BCPressure.RefPatch = 'bottom'
+vgf.Patch.left.BCPressure.Type = "DirEquilRefPatch"
+vgf.Patch.left.BCPressure.Cycle = "constant"
+vgf.Patch.left.BCPressure.RefGeom = "domain"
+vgf.Patch.left.BCPressure.RefPatch = "bottom"
 vgf.Patch.left.BCPressure.alltime.Value = 5.0
 
-vgf.Patch.right.BCPressure.Type = 'DirEquilRefPatch'
-vgf.Patch.right.BCPressure.Cycle = 'constant'
-vgf.Patch.right.BCPressure.RefGeom = 'domain'
-vgf.Patch.right.BCPressure.RefPatch = 'bottom'
+vgf.Patch.right.BCPressure.Type = "DirEquilRefPatch"
+vgf.Patch.right.BCPressure.Cycle = "constant"
+vgf.Patch.right.BCPressure.RefGeom = "domain"
+vgf.Patch.right.BCPressure.RefPatch = "bottom"
 vgf.Patch.right.BCPressure.alltime.Value = 3.0
 
-vgf.Patch.front.BCPressure.Type = 'FluxConst'
-vgf.Patch.front.BCPressure.Cycle = 'constant'
+vgf.Patch.front.BCPressure.Type = "FluxConst"
+vgf.Patch.front.BCPressure.Cycle = "constant"
 vgf.Patch.front.BCPressure.alltime.Value = 0.0
 
-vgf.Patch.back.BCPressure.Type = 'FluxConst'
-vgf.Patch.back.BCPressure.Cycle = 'constant'
+vgf.Patch.back.BCPressure.Type = "FluxConst"
+vgf.Patch.back.BCPressure.Cycle = "constant"
 vgf.Patch.back.BCPressure.alltime.Value = 0.0
 
-vgf.Patch.bottom.BCPressure.Type = 'FluxConst'
-vgf.Patch.bottom.BCPressure.Cycle = 'constant'
+vgf.Patch.bottom.BCPressure.Type = "FluxConst"
+vgf.Patch.bottom.BCPressure.Cycle = "constant"
 vgf.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-vgf.Patch.top.BCPressure.Type = 'FluxConst'
-vgf.Patch.top.BCPressure.Cycle = 'constant'
+vgf.Patch.top.BCPressure.Type = "FluxConst"
+vgf.Patch.top.BCPressure.Cycle = "constant"
 vgf.Patch.top.BCPressure.alltime.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-vgf.TopoSlopesX.Type = 'Constant'
-vgf.TopoSlopesX.GeomNames = 'domain'
+vgf.TopoSlopesX.Type = "Constant"
+vgf.TopoSlopesX.GeomNames = "domain"
 vgf.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-vgf.TopoSlopesY.Type = 'Constant'
-vgf.TopoSlopesY.GeomNames = 'domain'
+vgf.TopoSlopesY.Type = "Constant"
+vgf.TopoSlopesY.GeomNames = "domain"
 vgf.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-vgf.Mannings.Type = 'Constant'
-vgf.Mannings.GeomNames = 'domain'
-vgf.Mannings.Geom.domain.Value = 0.
+vgf.Mannings.Type = "Constant"
+vgf.Mannings.GeomNames = "domain"
+vgf.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-vgf.ICPressure.Type = 'HydroStaticPatch'
-vgf.ICPressure.GeomNames = 'domain'
+vgf.ICPressure.Type = "HydroStaticPatch"
+vgf.ICPressure.GeomNames = "domain"
 vgf.Geom.domain.ICPressure.Value = 3.0
-vgf.Geom.domain.ICPressure.RefGeom = 'domain'
-vgf.Geom.domain.ICPressure.RefPatch = 'bottom'
+vgf.Geom.domain.ICPressure.RefGeom = "domain"
+vgf.Geom.domain.ICPressure.RefPatch = "bottom"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vgf.PhaseSources.water.Type = 'Constant'
-vgf.PhaseSources.water.GeomNames = 'background'
+vgf.PhaseSources.water.Type = "Constant"
+vgf.PhaseSources.water.GeomNames = "background"
 vgf.PhaseSources.water.Geom.background.Value = 0.0
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vgf.KnownSolution = 'NoKnownSolution'
+vgf.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vgf.Solver = 'Richards'
+vgf.Solver = "Richards"
 vgf.Solver.MaxIter = 5
 
 vgf.Solver.Nonlinear.MaxIter = 10
 vgf.Solver.Nonlinear.ResidualTol = 1e-9
-vgf.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+vgf.Solver.Nonlinear.EtaChoice = "EtaConstant"
 vgf.Solver.Nonlinear.EtaValue = 1e-5
 vgf.Solver.Nonlinear.UseJacobian = True
 vgf.Solver.Nonlinear.DerivativeEpsilon = 1e-2
 
 vgf.Solver.Linear.KrylovDimension = 10
 
-vgf.Solver.Linear.Preconditioner = 'PFMG'
+vgf.Solver.Linear.Preconditioner = "PFMG"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-vgf.dist('van-genuchten-alpha.pfb')
-vgf.dist('van-genuchten-n.pfb')
-vgf.dist('van-genuchten-sr.pfb')
-vgf.dist('van-genuchten-ssat.pfb')
-
-vgf.run()
+# -----------------------------------------------------------------------------
+
+vgf.dist(new_output_dir_name + "/van-genuchten-alpha.pfb")
+vgf.dist(new_output_dir_name + "/van-genuchten-n.pfb")
+vgf.dist(new_output_dir_name + "/van-genuchten-sr.pfb")
+vgf.dist(new_output_dir_name + "/van-genuchten-ssat.pfb")
+
+
+vgf.run(working_directory=new_output_dir_name)
+
+passed = True
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/default_richards.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+for i in range(6):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/default_richards.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/default_richards.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/var_dz_1D.py b/test/python/var_dz_1D.py
index c159ebfe2..e57a3bd4c 100644
--- a/test/python/var_dz_1D.py
+++ b/test/python/var_dz_1D.py
@@ -1,14 +1,17 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Runs a simple sand draining problem, rectangular domain
 # with variable dz and a heterogenous subsurface with different K the top and bottom layers
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
+import sys
 from parflow import Run
 from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.compare import pf_test_file
 
-vardz = Run("var_d5z_1D", __file__)
+run_name = "var.dz.1d"
+vardz = Run(run_name, __file__)
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 vardz.FileVersion = 4
 
@@ -16,9 +19,9 @@
 vardz.Process.Topology.Q = 1
 vardz.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 vardz.ComputationalGrid.Lower.X = 0.0
 vardz.ComputationalGrid.Lower.Y = 0.0
@@ -32,28 +35,28 @@
 vardz.ComputationalGrid.NY = 1
 vardz.ComputationalGrid.NZ = 14
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-vardz.GeomInput.Names = 'domain_input het_input1 het_input2'
+vardz.GeomInput.Names = "domain_input het_input1 het_input2"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Geometry Input
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-vardz.GeomInput.domain_input.InputType = 'Box'
-vardz.GeomInput.domain_input.GeomName = 'domain'
+vardz.GeomInput.domain_input.InputType = "Box"
+vardz.GeomInput.domain_input.GeomName = "domain"
 
-vardz.GeomInput.het_input1.InputType = 'Box'
-vardz.GeomInput.het_input1.GeomName = 'het1'
+vardz.GeomInput.het_input1.InputType = "Box"
+vardz.GeomInput.het_input1.GeomName = "het1"
 
-vardz.GeomInput.het_input2.InputType = 'Box'
-vardz.GeomInput.het_input2.GeomName = 'het2'
+vardz.GeomInput.het_input2.InputType = "Box"
+vardz.GeomInput.het_input2.GeomName = "het2"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Geometry
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 vardz.Geom.domain.Lower.X = 0.0
 vardz.Geom.domain.Lower.Y = 0.0
@@ -63,7 +66,7 @@
 vardz.Geom.domain.Upper.Y = 1.0
 vardz.Geom.domain.Upper.Z = 1.4
 
-vardz.Geom.domain.Patches = 'left right front back bottom top'
+vardz.Geom.domain.Patches = "left right front back bottom top"
 
 vardz.Geom.het1.Lower.X = 0.0
 vardz.Geom.het1.Lower.Y = 0.0
@@ -81,13 +84,13 @@
 vardz.Geom.het2.Upper.Y = 1.0
 vardz.Geom.het2.Upper.Z = 0.1
 
-#--------------------------------------------
+# --------------------------------------------
 # variable dz assignments
-#------------------------------------------
+# ------------------------------------------
 
 vardz.Solver.Nonlinear.VariableDz = True
-vardz.dzScale.GeomNames = 'domain'
-vardz.dzScale.Type = 'nzList'
+vardz.dzScale.GeomNames = "domain"
+vardz.dzScale.Type = "nzList"
 vardz.dzScale.nzListNumber = 14
 vardz.Cell._0.dzScale.Value = 1.2
 vardz.Cell._1.dzScale.Value = 1.0
@@ -104,242 +107,278 @@
 vardz.Cell._12.dzScale.Value = 0.1
 vardz.Cell._13.dzScale.Value = 0.05
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vardz.Geom.Perm.Names = 'domain het1 het2'
+vardz.Geom.Perm.Names = "domain het1 het2"
 
-vardz.Geom.domain.Perm.Type = 'Constant'
+vardz.Geom.domain.Perm.Type = "Constant"
 vardz.Geom.domain.Perm.Value = 5.129
 
-vardz.Geom.het1.Perm.Type = 'Constant'
+vardz.Geom.het1.Perm.Type = "Constant"
 vardz.Geom.het1.Perm.Value = 0.0001
 
-vardz.Geom.het2.Perm.Type = 'Constant'
+vardz.Geom.het2.Perm.Type = "Constant"
 vardz.Geom.het2.Perm.Value = 0.001
 
-vardz.Perm.TensorType = 'TensorByGeom'
+vardz.Perm.TensorType = "TensorByGeom"
 
-vardz.Geom.Perm.TensorByGeom.Names = 'domain'
+vardz.Geom.Perm.TensorByGeom.Names = "domain"
 
 vardz.Geom.domain.Perm.TensorValX = 1.0
 vardz.Geom.domain.Perm.TensorValY = 1.0
 vardz.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vardz.SpecificStorage.Type = 'Constant'
-vardz.SpecificStorage.GeomNames = 'domain'
+vardz.SpecificStorage.Type = "Constant"
+vardz.SpecificStorage.GeomNames = "domain"
 vardz.Geom.domain.SpecificStorage.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vardz.Phase.Names = 'water'
+vardz.Phase.Names = "water"
 
-vardz.Phase.water.Density.Type = 'Constant'
+vardz.Phase.water.Density.Type = "Constant"
 vardz.Phase.water.Density.Value = 1.0
 
-vardz.Phase.water.Viscosity.Type = 'Constant'
+vardz.Phase.water.Viscosity.Type = "Constant"
 vardz.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vardz.Contaminants.Names = ''
+vardz.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vardz.Geom.Retardation.GeomNames = ''
+vardz.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 vardz.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 vardz.TimingInfo.BaseUnit = 1.0
 vardz.TimingInfo.StartCount = 0
 vardz.TimingInfo.StartTime = 0.0
 vardz.TimingInfo.StopTime = 50.0
 vardz.TimingInfo.DumpInterval = -100
-vardz.TimeStep.Type = 'Constant'
+vardz.TimeStep.Type = "Constant"
 vardz.TimeStep.Value = 0.01
 vardz.TimeStep.Value = 0.01
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vardz.Geom.Porosity.GeomNames = 'domain'
-vardz.Geom.domain.Porosity.Type = 'Constant'
+vardz.Geom.Porosity.GeomNames = "domain"
+vardz.Geom.domain.Porosity.Type = "Constant"
 vardz.Geom.domain.Porosity.Value = 0.4150
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vardz.Domain.GeomName = 'domain'
+vardz.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vardz.Phase.RelPerm.Type = 'VanGenuchten'
-vardz.Phase.RelPerm.GeomNames = 'domain'
+vardz.Phase.RelPerm.Type = "VanGenuchten"
+vardz.Phase.RelPerm.GeomNames = "domain"
 vardz.Geom.domain.RelPerm.Alpha = 2.7
 vardz.Geom.domain.RelPerm.N = 3.8
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-vardz.Phase.Saturation.Type = 'VanGenuchten'
-vardz.Phase.Saturation.GeomNames = 'domain'
+vardz.Phase.Saturation.Type = "VanGenuchten"
+vardz.Phase.Saturation.GeomNames = "domain"
 vardz.Geom.domain.Saturation.Alpha = 2.7
 vardz.Geom.domain.Saturation.N = 3.8
 vardz.Geom.domain.Saturation.SRes = 0.106
 vardz.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vardz.Wells.Names = ''
+vardz.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vardz.Cycle.Names = 'constant'
-vardz.Cycle.constant.Names = 'alltime'
+vardz.Cycle.Names = "constant"
+vardz.Cycle.constant.Names = "alltime"
 vardz.Cycle.constant.alltime.Length = 1
 vardz.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vardz.BCPressure.PatchNames = 'left right front back bottom top'
+vardz.BCPressure.PatchNames = "left right front back bottom top"
 
-vardz.Patch.left.BCPressure.Type = 'FluxConst'
-vardz.Patch.left.BCPressure.Cycle = 'constant'
-vardz.Patch.left.BCPressure.RefGeom = 'domain'
-vardz.Patch.left.BCPressure.RefPatch = 'bottom'
+vardz.Patch.left.BCPressure.Type = "FluxConst"
+vardz.Patch.left.BCPressure.Cycle = "constant"
+vardz.Patch.left.BCPressure.RefGeom = "domain"
+vardz.Patch.left.BCPressure.RefPatch = "bottom"
 vardz.Patch.left.BCPressure.alltime.Value = 0.0
 
-vardz.Patch.right.BCPressure.Type = 'FluxConst'
-vardz.Patch.right.BCPressure.Cycle = 'constant'
-vardz.Patch.right.BCPressure.RefGeom = 'domain'
-vardz.Patch.right.BCPressure.RefPatch = 'bottom'
+vardz.Patch.right.BCPressure.Type = "FluxConst"
+vardz.Patch.right.BCPressure.Cycle = "constant"
+vardz.Patch.right.BCPressure.RefGeom = "domain"
+vardz.Patch.right.BCPressure.RefPatch = "bottom"
 vardz.Patch.right.BCPressure.alltime.Value = 0.0
 
-vardz.Patch.front.BCPressure.Type = 'FluxConst'
-vardz.Patch.front.BCPressure.Cycle = 'constant'
+vardz.Patch.front.BCPressure.Type = "FluxConst"
+vardz.Patch.front.BCPressure.Cycle = "constant"
 vardz.Patch.front.BCPressure.alltime.Value = 0.0
 
-vardz.Patch.back.BCPressure.Type = 'FluxConst'
-vardz.Patch.back.BCPressure.Cycle = 'constant'
+vardz.Patch.back.BCPressure.Type = "FluxConst"
+vardz.Patch.back.BCPressure.Cycle = "constant"
 vardz.Patch.back.BCPressure.alltime.Value = 0.0
 
-vardz.Patch.bottom.BCPressure.Type = 'DirEquilRefPatch'
-vardz.Patch.bottom.BCPressure.Type = 'FluxConst'
-vardz.Patch.bottom.BCPressure.Cycle = 'constant'
-vardz.Patch.bottom.BCPressure.RefGeom = 'domain'
-vardz.Patch.bottom.BCPressure.RefPatch = 'bottom'
+vardz.Patch.bottom.BCPressure.Type = "DirEquilRefPatch"
+vardz.Patch.bottom.BCPressure.Type = "FluxConst"
+vardz.Patch.bottom.BCPressure.Cycle = "constant"
+vardz.Patch.bottom.BCPressure.RefGeom = "domain"
+vardz.Patch.bottom.BCPressure.RefPatch = "bottom"
 vardz.Patch.bottom.BCPressure.alltime.Value = 0.0
 
-vardz.Patch.top.BCPressure.Type = 'DirEquilRefPatch'
-vardz.Patch.top.BCPressure.Type = 'FluxConst'
-vardz.Patch.top.BCPressure.Cycle = 'constant'
-vardz.Patch.top.BCPressure.RefGeom = 'domain'
-vardz.Patch.top.BCPressure.RefPatch = 'bottom'
+vardz.Patch.top.BCPressure.Type = "DirEquilRefPatch"
+vardz.Patch.top.BCPressure.Type = "FluxConst"
+vardz.Patch.top.BCPressure.Cycle = "constant"
+vardz.Patch.top.BCPressure.RefGeom = "domain"
+vardz.Patch.top.BCPressure.RefPatch = "bottom"
 vardz.Patch.top.BCPressure.alltime.Value = -0.0001
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-vardz.TopoSlopesX.Type = 'Constant'
-vardz.TopoSlopesX.GeomNames = 'domain'
+vardz.TopoSlopesX.Type = "Constant"
+vardz.TopoSlopesX.GeomNames = "domain"
 vardz.TopoSlopesX.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-vardz.TopoSlopesY.Type = 'Constant'
-vardz.TopoSlopesY.GeomNames = 'domain'
+vardz.TopoSlopesY.Type = "Constant"
+vardz.TopoSlopesY.GeomNames = "domain"
 vardz.TopoSlopesY.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Mannings coefficient
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-vardz.Mannings.Type = 'Constant'
-vardz.Mannings.GeomNames = 'domain'
-vardz.Mannings.Geom.domain.Value = 0.
+vardz.Mannings.Type = "Constant"
+vardz.Mannings.GeomNames = "domain"
+vardz.Mannings.Geom.domain.Value = 0.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-vardz.ICPressure.Type = 'Constant'
-vardz.ICPressure.GeomNames = 'domain'
+vardz.ICPressure.Type = "Constant"
+vardz.ICPressure.GeomNames = "domain"
 vardz.Geom.domain.ICPressure.Value = -10.0
-vardz.Geom.domain.ICPressure.RefGeom = 'domain'
-vardz.Geom.domain.ICPressure.RefPatch = 'top'
+vardz.Geom.domain.ICPressure.RefGeom = "domain"
+vardz.Geom.domain.ICPressure.RefPatch = "top"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vardz.PhaseSources.water.Type = 'Constant'
-vardz.PhaseSources.water.GeomNames = 'domain'
+vardz.PhaseSources.water.Type = "Constant"
+vardz.PhaseSources.water.GeomNames = "domain"
 vardz.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vardz.KnownSolution = 'NoKnownSolution'
+vardz.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-vardz.Solver = 'Richards'
+vardz.Solver = "Richards"
 vardz.Solver.MaxIter = 2500
 
 vardz.Solver.Nonlinear.MaxIter = 200
 vardz.Solver.Nonlinear.ResidualTol = 1e-9
-vardz.Solver.Nonlinear.EtaChoice = 'Walker1'
+vardz.Solver.Nonlinear.EtaChoice = "Walker1"
 vardz.Solver.Nonlinear.EtaValue = 1e-5
 vardz.Solver.Nonlinear.UseJacobian = True
 vardz.Solver.Nonlinear.DerivativeEpsilon = 1e-10
 
 vardz.Solver.Linear.KrylovDimension = 10
 
-vardz.Solver.Linear.Preconditioner = 'MGSemi'
-vardz.Solver.Linear.Preconditioner = 'PFMG'
+vardz.Solver.Linear.Preconditioner = "MGSemi"
+vardz.Solver.Linear.Preconditioner = "PFMG"
 vardz.Solver.Linear.Preconditioner.MGSemi.MaxIter = 1
 vardz.Solver.Linear.Preconditioner.MGSemi.MaxLevels = 10
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and do tests
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/vardz')
-mkdir(dir_name)
-vardz.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/var_d5z_1D")
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+vardz.run(working_directory=new_output_dir_name)
+
+passed = True
+
+test_files = ["perm_x", "perm_y", "perm_z"]
+for test_file in test_files:
+    filename = f"/{run_name}.out.{test_file}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in {test_file}",
+    ):
+        passed = False
+
+for i in range(0, 30, 5):
+    timestep = str(i).rjust(5, "0")
+    filename = f"/{run_name}.out.press.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Pressure for timestep {timestep}",
+    ):
+        passed = False
+    filename = f"/{run_name}.out.satur.{timestep}.pfb"
+    if not pf_test_file(
+        new_output_dir_name + filename,
+        correct_output_dir_name + filename,
+        f"Max difference in Saturation for timestep {timestep}",
+    ):
+        passed = False
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
+    sys.exit(1)
diff --git a/test/python/washita/Dist_Forcings.py b/test/python/washita/Dist_Forcings.py
index bbd68e7e8..bf3e3fac1 100644
--- a/test/python/washita/Dist_Forcings.py
+++ b/test/python/washita/Dist_Forcings.py
@@ -1,27 +1,28 @@
 # Import the ParFlow TCL package
 #
 from parflow import Run
+
 Dist_Forcings = Run("Dist_Forcings", __file__)
 
 
-#set tcl_precision 16
+# set tcl_precision 16
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # File input version number
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 Dist_Forcings.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Process Topology
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 Dist_Forcings.Process.Topology.P = 1
 Dist_Forcings.Process.Topology.Q = 1
 Dist_Forcings.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 Dist_Forcings.ComputationalGrid.Lower.X = 0.0
 Dist_Forcings.ComputationalGrid.Lower.Y = 0.0
 Dist_Forcings.ComputationalGrid.Lower.Z = 0.0
@@ -34,8 +35,8 @@
 Dist_Forcings.ComputationalGrid.NY = 41
 Dist_Forcings.ComputationalGrid.NZ = 24
 
-name = 'NLDAS'
-var = [list "APCP" "DLWR" "DSWR" "Press" "SPFH" "Temp" "UGRD" "VGRD"]
+name = "NLDAS"
+# var = ["APCP" "DLWR" "DSWR" "Press" "SPFH" "Temp" "UGRD" "VGRD"]
 
 end = 4
 #   for {set i 0} {$i <= $end} {incr i} {
diff --git a/test/python/washita/LW_NetCDF_Test.py b/test/python/washita/LW_NetCDF_Test.py
index 44b474eb4..2e3a1c773 100644
--- a/test/python/washita/LW_NetCDF_Test.py
+++ b/test/python/washita/LW_NetCDF_Test.py
@@ -7,18 +7,18 @@
 # package   require parflow
 LW_NetCDF_Test = Run("LW_NetCDF_Test", __file__)
 
-LW_NetCDF_Test. = 'FileVersion 4'
+LW_NetCDF_Test.FileVersion = "FileVersion 4"
 
-#-----------------------------------------------------------------------------
-# Set Processor topology 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+# Set Processor topology
+# -----------------------------------------------------------------------------
 LW_NetCDF_Test.Process.Topology.P = 1
 LW_NetCDF_Test.Process.Topology.Q = 1
 LW_NetCDF_Test.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Make a directory for the simulation and copy inputs into it
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # exec mkdir "Outputs"
 # cd "./Outputs"
 
@@ -27,17 +27,17 @@
 # file copy -force "../../parflow_input/IndicatorFile_Gleeson.50z.pfb"   .
 # file copy -force "../../parflow_input/press.init.nc"  .
 
-#CLM Inputs
+# CLM Inputs
 # file copy -force "../../clm_input/drv_clmin.dat" .
 # file copy -force "../../clm_input/drv_vegp.dat"  .
-# file copy -force "../../clm_input/drv_vegm.alluv.dat"  . 
-# file copy -force "../../clm_input/metForcing.nc"  . 
+# file copy -force "../../clm_input/drv_vegm.alluv.dat"  .
+# file copy -force "../../clm_input/metForcing.nc"  .
 
 # puts "Files Copied"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 LW_NetCDF_Test.ComputationalGrid.Lower.X = 0.0
 LW_NetCDF_Test.ComputationalGrid.Lower.Y = 0.0
 LW_NetCDF_Test.ComputationalGrid.Lower.Z = 0.0
@@ -51,35 +51,37 @@
 LW_NetCDF_Test.ComputationalGrid.NZ = 50
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Names of the GeomInputs
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.GeomInput.Names = 'box_input indi_input'
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.GeomInput.Names = "box_input indi_input"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.GeomInput.box_input.InputType = 'Box'
-LW_NetCDF_Test.GeomInput.box_input.GeomName = 'domain'
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.GeomInput.box_input.InputType = "Box"
+LW_NetCDF_Test.GeomInput.box_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
-# Domain Geometry 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+# Domain Geometry
+# -----------------------------------------------------------------------------
 LW_NetCDF_Test.Geom.domain.Lower.X = 0.0
 LW_NetCDF_Test.Geom.domain.Lower.Y = 0.0
 LW_NetCDF_Test.Geom.domain.Lower.Z = 0.0
-#  
+#
 LW_NetCDF_Test.Geom.domain.Upper.X = 41000.0
 LW_NetCDF_Test.Geom.domain.Upper.Y = 41000.0
 LW_NetCDF_Test.Geom.domain.Upper.Z = 100.0
-LW_NetCDF_Test.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+LW_NetCDF_Test.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Indicator Geometry Input
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.GeomInput.indi_input.InputType = 'IndicatorField'
-LW_NetCDF_Test.GeomInput.indi_input.GeomNames = 's1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 g1 g2 g3 g4 g5 g6 g7 g8'
-LW_NetCDF_Test.Geom.indi_input.FileName = 'IndicatorFile_Gleeson.50z.pfb'
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.GeomInput.indi_input.InputType = "IndicatorField"
+LW_NetCDF_Test.GeomInput.indi_input.GeomNames = (
+    "s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 g1 g2 g3 g4 g5 g6 g7 g8"
+)
+LW_NetCDF_Test.Geom.indi_input.FileName = "IndicatorFile_Gleeson.50z.pfb"
 
 LW_NetCDF_Test.GeomInput.s1.Value = 1
 LW_NetCDF_Test.GeomInput.s2.Value = 2
@@ -103,210 +105,210 @@
 LW_NetCDF_Test.GeomInput.g7.Value = 27
 LW_NetCDF_Test.GeomInput.g8.Value = 28
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Permeability (values in m/hr)
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.Geom.Perm.Names = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9 g2 g3 g6 g8'
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.Geom.Perm.Names = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9 g2 g3 g6 g8"
 
-LW_NetCDF_Test.Geom.domain.Perm.Type = 'Constant'
+LW_NetCDF_Test.Geom.domain.Perm.Type = "Constant"
 LW_NetCDF_Test.Geom.domain.Perm.Value = 0.2
 
-LW_NetCDF_Test.Geom.s1.Perm.Type = 'Constant'
+LW_NetCDF_Test.Geom.s1.Perm.Type = "Constant"
 LW_NetCDF_Test.Geom.s1.Perm.Value = 0.269022595
 
-LW_NetCDF_Test.Geom.s2.Perm.Type = 'Constant'
+LW_NetCDF_Test.Geom.s2.Perm.Type = "Constant"
 LW_NetCDF_Test.Geom.s2.Perm.Value = 0.043630356
 
-LW_NetCDF_Test.Geom.s3.Perm.Type = 'Constant'
+LW_NetCDF_Test.Geom.s3.Perm.Type = "Constant"
 LW_NetCDF_Test.Geom.s3.Perm.Value = 0.015841225
 
-LW_NetCDF_Test.Geom.s4.Perm.Type = 'Constant'
+LW_NetCDF_Test.Geom.s4.Perm.Type = "Constant"
 LW_NetCDF_Test.Geom.s4.Perm.Value = 0.007582087
 
-LW_NetCDF_Test.Geom.s5.Perm.Type = 'Constant'
+LW_NetCDF_Test.Geom.s5.Perm.Type = "Constant"
 LW_NetCDF_Test.Geom.s5.Perm.Value = 0.01818816
 
-LW_NetCDF_Test.Geom.s6.Perm.Type = 'Constant'
+LW_NetCDF_Test.Geom.s6.Perm.Type = "Constant"
 LW_NetCDF_Test.Geom.s6.Perm.Value = 0.005009435
 
-LW_NetCDF_Test.Geom.s7.Perm.Type = 'Constant'
+LW_NetCDF_Test.Geom.s7.Perm.Type = "Constant"
 LW_NetCDF_Test.Geom.s7.Perm.Value = 0.005492736
 
-LW_NetCDF_Test.Geom.s8.Perm.Type = 'Constant'
+LW_NetCDF_Test.Geom.s8.Perm.Type = "Constant"
 LW_NetCDF_Test.Geom.s8.Perm.Value = 0.004675077
 
-LW_NetCDF_Test.Geom.s9.Perm.Type = 'Constant'
+LW_NetCDF_Test.Geom.s9.Perm.Type = "Constant"
 LW_NetCDF_Test.Geom.s9.Perm.Value = 0.003386794
 
-LW_NetCDF_Test.Geom.g2.Perm.Type = 'Constant'
+LW_NetCDF_Test.Geom.g2.Perm.Type = "Constant"
 LW_NetCDF_Test.Geom.g2.Perm.Value = 0.025
 
-LW_NetCDF_Test.Geom.g3.Perm.Type = 'Constant'
+LW_NetCDF_Test.Geom.g3.Perm.Type = "Constant"
 LW_NetCDF_Test.Geom.g3.Perm.Value = 0.059
 
-LW_NetCDF_Test.Geom.g6.Perm.Type = 'Constant'
+LW_NetCDF_Test.Geom.g6.Perm.Type = "Constant"
 LW_NetCDF_Test.Geom.g6.Perm.Value = 0.2
 
-LW_NetCDF_Test.Geom.g8.Perm.Type = 'Constant'
+LW_NetCDF_Test.Geom.g8.Perm.Type = "Constant"
 LW_NetCDF_Test.Geom.g8.Perm.Value = 0.68
 
-LW_NetCDF_Test.Perm.TensorType = 'TensorByGeom'
-LW_NetCDF_Test.Geom.Perm.TensorByGeom.Names = 'domain'
-LW_NetCDF_Test.Geom.domain.Perm.TensorValX = 1.0d0
-LW_NetCDF_Test.Geom.domain.Perm.TensorValY = 1.0d0
-LW_NetCDF_Test.Geom.domain.Perm.TensorValZ = 1.0d0
+LW_NetCDF_Test.Perm.TensorType = "TensorByGeom"
+LW_NetCDF_Test.Geom.Perm.TensorByGeom.Names = "domain"
+LW_NetCDF_Test.Geom.domain.Perm.TensorValX = 1.0
+LW_NetCDF_Test.Geom.domain.Perm.TensorValY = 1.0
+LW_NetCDF_Test.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.SpecificStorage.Type = 'Constant'
-LW_NetCDF_Test.SpecificStorage.GeomNames = 'domain'
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.SpecificStorage.Type = "Constant"
+LW_NetCDF_Test.SpecificStorage.GeomNames = "domain"
 LW_NetCDF_Test.Geom.domain.SpecificStorage.Value = 1.0e-5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.Phase.Names = 'water'
-LW_NetCDF_Test.Phase.water.Density.Type = 'Constant'
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.Phase.Names = "water"
+LW_NetCDF_Test.Phase.water.Density.Type = "Constant"
 LW_NetCDF_Test.Phase.water.Density.Value = 1.0
-LW_NetCDF_Test.Phase.water.Viscosity.Type = 'Constant'
+LW_NetCDF_Test.Phase.water.Viscosity.Type = "Constant"
 LW_NetCDF_Test.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.Contaminants.Names = ''
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 LW_NetCDF_Test.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Timing (time units is set by units of permeability)
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 LW_NetCDF_Test.TimingInfo.BaseUnit = 1.0
 LW_NetCDF_Test.TimingInfo.StartCount = 0.0
 LW_NetCDF_Test.TimingInfo.StartTime = 0.0
 LW_NetCDF_Test.TimingInfo.StopTime = 72.0
 LW_NetCDF_Test.TimingInfo.DumpInterval = 3.0
-LW_NetCDF_Test.TimeStep.Type = 'Constant'
+LW_NetCDF_Test.TimeStep.Type = "Constant"
 LW_NetCDF_Test.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.Geom.Porosity.GeomNames = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9'
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.Geom.Porosity.GeomNames = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9"
 
-LW_NetCDF_Test.Geom.domain.Porosity.Type = 'Constant'
+LW_NetCDF_Test.Geom.domain.Porosity.Type = "Constant"
 LW_NetCDF_Test.Geom.domain.Porosity.Value = 0.4
 
-LW_NetCDF_Test.Geom.s1.Porosity.Type = 'Constant'
+LW_NetCDF_Test.Geom.s1.Porosity.Type = "Constant"
 LW_NetCDF_Test.Geom.s1.Porosity.Value = 0.375
 
-LW_NetCDF_Test.Geom.s2.Porosity.Type = 'Constant'
+LW_NetCDF_Test.Geom.s2.Porosity.Type = "Constant"
 LW_NetCDF_Test.Geom.s2.Porosity.Value = 0.39
 
-LW_NetCDF_Test.Geom.s3.Porosity.Type = 'Constant'
+LW_NetCDF_Test.Geom.s3.Porosity.Type = "Constant"
 LW_NetCDF_Test.Geom.s3.Porosity.Value = 0.387
 
-LW_NetCDF_Test.Geom.s4.Porosity.Type = 'Constant'
+LW_NetCDF_Test.Geom.s4.Porosity.Type = "Constant"
 LW_NetCDF_Test.Geom.s4.Porosity.Value = 0.439
 
-LW_NetCDF_Test.Geom.s5.Porosity.Type = 'Constant'
+LW_NetCDF_Test.Geom.s5.Porosity.Type = "Constant"
 LW_NetCDF_Test.Geom.s5.Porosity.Value = 0.489
 
-LW_NetCDF_Test.Geom.s6.Porosity.Type = 'Constant'
+LW_NetCDF_Test.Geom.s6.Porosity.Type = "Constant"
 LW_NetCDF_Test.Geom.s6.Porosity.Value = 0.399
 
-LW_NetCDF_Test.Geom.s7.Porosity.Type = 'Constant'
+LW_NetCDF_Test.Geom.s7.Porosity.Type = "Constant"
 LW_NetCDF_Test.Geom.s7.Porosity.Value = 0.384
 
-LW_NetCDF_Test.Geom.s8.Porosity.Type = 'Constant'
+LW_NetCDF_Test.Geom.s8.Porosity.Type = "Constant"
 LW_NetCDF_Test.Geom.s8.Porosity.Value = 0.482
 
-LW_NetCDF_Test.Geom.s9.Porosity.Type = 'Constant'
+LW_NetCDF_Test.Geom.s9.Porosity.Type = "Constant"
 LW_NetCDF_Test.Geom.s9.Porosity.Value = 0.442
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.Domain.GeomName = 'domain'
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.Domain.GeomName = "domain"
 
-#----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 # Mobility
-#----------------------------------------------------------------------------
-LW_NetCDF_Test.Phase.water.Mobility.Type = 'Constant'
+# ----------------------------------------------------------------------------
+LW_NetCDF_Test.Phase.water.Mobility.Type = "Constant"
 LW_NetCDF_Test.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.Wells.Names = ''
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.Cycle.Names = 'constant'
-LW_NetCDF_Test.Cycle.constant.Names = 'alltime'
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.Cycle.Names = "constant"
+LW_NetCDF_Test.Cycle.constant.Names = "alltime"
 LW_NetCDF_Test.Cycle.constant.alltime.Length = 1
 LW_NetCDF_Test.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.BCPressure.PatchNames = [pfget Geom.domain.Patches]
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.BCPressure.PatchNames = LW_NetCDF_Test.Geom.domain.Patches
 
-LW_NetCDF_Test.Patch.x_lower.BCPressure.Type = 'FluxConst'
-LW_NetCDF_Test.Patch.x_lower.BCPressure.Cycle = 'constant'
+LW_NetCDF_Test.Patch.x_lower.BCPressure.Type = "FluxConst"
+LW_NetCDF_Test.Patch.x_lower.BCPressure.Cycle = "constant"
 LW_NetCDF_Test.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-LW_NetCDF_Test.Patch.y_lower.BCPressure.Type = 'FluxConst'
-LW_NetCDF_Test.Patch.y_lower.BCPressure.Cycle = 'constant'
+LW_NetCDF_Test.Patch.y_lower.BCPressure.Type = "FluxConst"
+LW_NetCDF_Test.Patch.y_lower.BCPressure.Cycle = "constant"
 LW_NetCDF_Test.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-LW_NetCDF_Test.Patch.z_lower.BCPressure.Type = 'FluxConst'
-LW_NetCDF_Test.Patch.z_lower.BCPressure.Cycle = 'constant'
+LW_NetCDF_Test.Patch.z_lower.BCPressure.Type = "FluxConst"
+LW_NetCDF_Test.Patch.z_lower.BCPressure.Cycle = "constant"
 LW_NetCDF_Test.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-LW_NetCDF_Test.Patch.x_upper.BCPressure.Type = 'FluxConst'
-LW_NetCDF_Test.Patch.x_upper.BCPressure.Cycle = 'constant'
+LW_NetCDF_Test.Patch.x_upper.BCPressure.Type = "FluxConst"
+LW_NetCDF_Test.Patch.x_upper.BCPressure.Cycle = "constant"
 LW_NetCDF_Test.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-LW_NetCDF_Test.Patch.y_upper.BCPressure.Type = 'FluxConst'
-LW_NetCDF_Test.Patch.y_upper.BCPressure.Cycle = 'constant'
+LW_NetCDF_Test.Patch.y_upper.BCPressure.Type = "FluxConst"
+LW_NetCDF_Test.Patch.y_upper.BCPressure.Cycle = "constant"
 LW_NetCDF_Test.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-LW_NetCDF_Test.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-LW_NetCDF_Test.Patch.z_upper.BCPressure.Cycle = 'constant'
+LW_NetCDF_Test.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+LW_NetCDF_Test.Patch.z_upper.BCPressure.Cycle = "constant"
 LW_NetCDF_Test.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Topo slopes in x-direction
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.TopoSlopesX.Type = 'NCFile'
-LW_NetCDF_Test.TopoSlopesX.GeomNames = 'domain'
-LW_NetCDF_Test.TopoSlopesX.FileName = 'slopes.nc'
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.TopoSlopesX.Type = "NCFile"
+LW_NetCDF_Test.TopoSlopesX.GeomNames = "domain"
+LW_NetCDF_Test.TopoSlopesX.FileName = "slopes.nc"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Topo slopes in y-direction
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.TopoSlopesY.Type = 'NCFile'
-LW_NetCDF_Test.TopoSlopesY.GeomNames = 'domain'
-LW_NetCDF_Test.TopoSlopesY.FileName = 'slopes.nc'
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.TopoSlopesY.Type = "NCFile"
+LW_NetCDF_Test.TopoSlopesY.GeomNames = "domain"
+LW_NetCDF_Test.TopoSlopesY.FileName = "slopes.nc"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mannings coefficient
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.Mannings.Type = 'Constant'
-LW_NetCDF_Test.Mannings.GeomNames = 'domain'
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.Mannings.Type = "Constant"
+LW_NetCDF_Test.Mannings.GeomNames = "domain"
 LW_NetCDF_Test.Mannings.Geom.domain.Value = 5.52e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.Phase.RelPerm.Type = 'VanGenuchten'
-LW_NetCDF_Test.Phase.RelPerm.GeomNames = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9 '
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.Phase.RelPerm.Type = "VanGenuchten"
+LW_NetCDF_Test.Phase.RelPerm.GeomNames = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9 "
 
 LW_NetCDF_Test.Geom.domain.RelPerm.Alpha = 3.5
 LW_NetCDF_Test.Geom.domain.RelPerm.N = 2.0
@@ -338,14 +340,14 @@
 LW_NetCDF_Test.Geom.s9.RelPerm.Alpha = 1.585
 LW_NetCDF_Test.Geom.s9.RelPerm.N = 2.413
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Saturation
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.Phase.Saturation.Type = 'VanGenuchten'
-LW_NetCDF_Test.Phase.Saturation.GeomNames = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9 '
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.Phase.Saturation.Type = "VanGenuchten"
+LW_NetCDF_Test.Phase.Saturation.GeomNames = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9 "
 
 LW_NetCDF_Test.Geom.domain.Saturation.Alpha = 3.5
-LW_NetCDF_Test.Geom.domain.Saturation.N = 2.
+LW_NetCDF_Test.Geom.domain.Saturation.N = 2.0
 LW_NetCDF_Test.Geom.domain.Saturation.SRes = 0.2
 LW_NetCDF_Test.Geom.domain.Saturation.SSat = 1.0
 
@@ -394,47 +396,47 @@
 LW_NetCDF_Test.Geom.s9.Saturation.SRes = 0.000001
 LW_NetCDF_Test.Geom.s9.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.PhaseSources.water.Type = 'Constant'
-LW_NetCDF_Test.PhaseSources.water.GeomNames = 'domain'
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.PhaseSources.water.Type = "Constant"
+LW_NetCDF_Test.PhaseSources.water.GeomNames = "domain"
 LW_NetCDF_Test.PhaseSources.water.Geom.domain.Value = 0.0
 
-#----------------------------------------------------------------
+# ----------------------------------------------------------------
 # CLM Settings:
 # ------------------------------------------------------------
-LW_NetCDF_Test.Solver.LSM = 'CLM'
-LW_NetCDF_Test.Solver.CLM.CLMFileDir = 'clm_output/'
+LW_NetCDF_Test.Solver.LSM = "CLM"
+LW_NetCDF_Test.Solver.CLM.CLMFileDir = "clm_output/"
 LW_NetCDF_Test.Solver.CLM.Print1dOut = False
 LW_NetCDF_Test.Solver.BinaryOutDir = False
 LW_NetCDF_Test.Solver.CLM.CLMDumpInterval = 1
 
-LW_NetCDF_Test.Solver.CLM.MetForcing = 'NC'
-LW_NetCDF_Test.Solver.CLM.MetFileName = 'metForcing.nc'
-#pfset Solver.CLM.MetFilePath                          "../../NLDAS/"
+LW_NetCDF_Test.Solver.CLM.MetForcing = "NC"
+LW_NetCDF_Test.Solver.CLM.MetFileName = "metForcing.nc"
+# pfset Solver.CLM.MetFilePath                          "../../NLDAS/"
 LW_NetCDF_Test.Solver.CLM.MetFileNT = 1
 LW_NetCDF_Test.Solver.CLM.IstepStart = 1
 
-LW_NetCDF_Test.Solver.CLM.EvapBeta = 'Linear'
-LW_NetCDF_Test.Solver.CLM.VegWaterStress = 'Saturation'
+LW_NetCDF_Test.Solver.CLM.EvapBeta = "Linear"
+LW_NetCDF_Test.Solver.CLM.VegWaterStress = "Saturation"
 LW_NetCDF_Test.Solver.CLM.ResSat = 0.1
 LW_NetCDF_Test.Solver.CLM.WiltingPoint = 0.12
 LW_NetCDF_Test.Solver.CLM.FieldCapacity = 0.98
-LW_NetCDF_Test.Solver.CLM.IrrigationType = 'none'
+LW_NetCDF_Test.Solver.CLM.IrrigationType = "none"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
-LW_NetCDF_Test.ICPressure.Type = 'NCFile'
-LW_NetCDF_Test.ICPressure.GeomNames = 'domain'
-LW_NetCDF_Test.Geom.domain.ICPressure.RefPatch = 'z_upper'
-LW_NetCDF_Test.Geom.domain.ICPressure.FileName = 'press.init.nc'
+# ---------------------------------------------------------
+LW_NetCDF_Test.ICPressure.Type = "NCFile"
+LW_NetCDF_Test.ICPressure.GeomNames = "domain"
+LW_NetCDF_Test.Geom.domain.ICPressure.RefPatch = "z_upper"
+LW_NetCDF_Test.Geom.domain.ICPressure.FileName = "press.init.nc"
 
-#----------------------------------------------------------------
+# ----------------------------------------------------------------
 # Outputs
 # ------------------------------------------------------------
-#Writing output (all pfb):
+# Writing output (all pfb):
 LW_NetCDF_Test.Solver.PrintSubsurfData = False
 LW_NetCDF_Test.Solver.PrintPressure = False
 LW_NetCDF_Test.Solver.PrintSaturation = False
@@ -455,38 +457,38 @@
 LW_NetCDF_Test.Solver.WriteSiloCLM = False
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
-LW_NetCDF_Test.KnownSolution = 'NoKnownSolution'
+# -----------------------------------------------------------------------------
+LW_NetCDF_Test.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # ParFlow Solution
-LW_NetCDF_Test.Solver = 'Richards'
+LW_NetCDF_Test.Solver = "Richards"
 LW_NetCDF_Test.Solver.TerrainFollowingGrid = True
 LW_NetCDF_Test.Solver.Nonlinear.VariableDz = False
 
 LW_NetCDF_Test.Solver.MaxIter = 25000
-LW_NetCDF_Test.Solver.Drop = 1E-20
-LW_NetCDF_Test.Solver.AbsTol = 1E-8
+LW_NetCDF_Test.Solver.Drop = 1e-20
+LW_NetCDF_Test.Solver.AbsTol = 1e-8
 LW_NetCDF_Test.Solver.MaxConvergenceFailures = 8
 LW_NetCDF_Test.Solver.Nonlinear.MaxIter = 80
 LW_NetCDF_Test.Solver.Nonlinear.ResidualTol = 1e-6
 
 ## new solver settings for Terrain Following Grid
-LW_NetCDF_Test.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+LW_NetCDF_Test.Solver.Nonlinear.EtaChoice = "EtaConstant"
 LW_NetCDF_Test.Solver.Nonlinear.EtaValue = 0.001
 LW_NetCDF_Test.Solver.Nonlinear.UseJacobian = True
 LW_NetCDF_Test.Solver.Nonlinear.DerivativeEpsilon = 1e-16
 LW_NetCDF_Test.Solver.Nonlinear.StepTol = 1e-30
-LW_NetCDF_Test.Solver.Nonlinear.Globalization = 'LineSearch'
+LW_NetCDF_Test.Solver.Nonlinear.Globalization = "LineSearch"
 LW_NetCDF_Test.Solver.Linear.KrylovDimension = 70
 LW_NetCDF_Test.Solver.Linear.MaxRestarts = 2
 
-LW_NetCDF_Test.Solver.Linear.Preconditioner = 'PFMG'
-LW_NetCDF_Test.Solver.Linear.Preconditioner.PCMatrixType = 'FullJacobian'
+LW_NetCDF_Test.Solver.Linear.Preconditioner = "PFMG"
+LW_NetCDF_Test.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
 
 LW_NetCDF_Test.NetCDF.NumStepsPerFile = 5
 LW_NetCDF_Test.NetCDF.CLMNumStepsPerFile = 24
@@ -503,18 +505,18 @@
 LW_NetCDF_Test.NetCDF.WriteOverlandBCFlux = True
 LW_NetCDF_Test.NetCDF.WriteCLM = True
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Distribute inputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 LW_NetCDF_Test.ComputationalGrid.NX = 41
 LW_NetCDF_Test.ComputationalGrid.NY = 41
 LW_NetCDF_Test.ComputationalGrid.NZ = 50
 
 # pfdist IndicatorFile_Gleeson.50z.pfb
 
-#-----------------------------------------------------------------------------
-# Run Simulation 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+# Run Simulation
+# -----------------------------------------------------------------------------
 # set runname "LW"
 # puts $runname
 # pfrun    $runname
@@ -523,11 +525,11 @@
 ##-----------------------------------------------------------------------------
 ## Undistribute outputs
 ##-----------------------------------------------------------------------------
-#pfundist $runname
-#pfundist press.init.pfb
-#pfundist LW.slopex.pfb
-#pfundist LW.slopey.pfb
-#pfundist IndicatorFile_Gleeson.50z.pfb
+# pfundist $runname
+# pfundist press.init.pfb
+# pfundist LW.slopex.pfb
+# pfundist LW.slopey.pfb
+# pfundist IndicatorFile_Gleeson.50z.pfb
 #
 # puts "ParFlow run Complete"
 
diff --git a/test/python/washita/LW_Test.py b/test/python/washita/LW_Test.py
index 1302d0966..72de32862 100644
--- a/test/python/washita/LW_Test.py
+++ b/test/python/washita/LW_Test.py
@@ -1,31 +1,35 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # SCRIPT TO RUN LITTLE WASHITA DOMAIN WITH TERRAIN-FOLLOWING GRID
 # DETAILS:
 # Arugments are 1) runname 2) year
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
+import os
+import numpy as np
 from parflow import Run
 from parflow.tools.fs import mkdir, cp, chdir, get_absolute_path
+from parflow.tools.io import read_pfb, write_pfb
+
 
 LW_Test = Run("LW_Test", __file__)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Make a directory for the simulation run and copy files
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-dir_name = get_absolute_path('test_output/LW_test')
+dir_name = get_absolute_path("test_output/LW_test")
 mkdir(dir_name)
 
-cp('$PF_SRC/test/tcl/washita/clm_input/drv_clmin.dat', dir_name)
-cp('$PF_SRC/test/tcl/washita/clm_input/drv_vegm.alluv.dat', dir_name)
-cp('$PF_SRC/test/tcl/washita/clm_input/drv_vegp.dat', dir_name)
-cp('$PF_SRC/test/tcl/washita/parflow_input/LW.slopex.pfb', dir_name)
-cp('$PF_SRC/test/tcl/washita/parflow_input/LW.slopey.pfb', dir_name)
-cp('$PF_SRC/test/tcl/washita/parflow_input/IndicatorFile_Gleeson.50z.pfb', dir_name)
-cp('$PF_SRC/test/tcl/washita/parflow_input/press.init.pfb', dir_name)
+cp("$PF_SRC/test/tcl/washita/clm_input/drv_clmin.dat", dir_name)
+cp("$PF_SRC/test/tcl/washita/clm_input/drv_vegm.alluv.dat", dir_name)
+cp("$PF_SRC/test/tcl/washita/clm_input/drv_vegp.dat", dir_name)
+cp("$PF_SRC/test/tcl/washita/parflow_input/LW.slopex.pfb", dir_name)
+cp("$PF_SRC/test/tcl/washita/parflow_input/LW.slopey.pfb", dir_name)
+cp("$PF_SRC/test/tcl/washita/parflow_input/IndicatorFile_Gleeson.50z.pfb", dir_name)
+cp("$PF_SRC/test/tcl/washita/parflow_input/press.init.pfb", dir_name)
 
 
-#nldas_files = [
+# nldas_files = [
 #  'NLDAS.DSWR.000001_to_000024.pfb',
 #  'NLDAS.DLWR.000001_to_000024.pfb',
 #  'NLDAS.APCP.000001_to_000024.pfb',
@@ -36,24 +40,24 @@
 #  'NLDAS.SPFH.000001_to_000024.pfb'
 #  ]
 #
-#for file in nldas_files:
+# for file in nldas_files:
 #  cp('$PF_SRC/test/tcl/washita/NLDAS/' + file, dir_name)
 #  LW_Test.dist(dir_name + '/' + file)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_Test.FileVersion = 4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set Processor topology
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 LW_Test.Process.Topology.P = 1
 LW_Test.Process.Topology.Q = 1
 LW_Test.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_Test.ComputationalGrid.Lower.X = 0.0
 LW_Test.ComputationalGrid.Lower.Y = 0.0
@@ -67,22 +71,22 @@
 LW_Test.ComputationalGrid.NY = 41
 LW_Test.ComputationalGrid.NZ = 50
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Names of the GeomInputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.GeomInput.Names = 'box_input indi_input'
+LW_Test.GeomInput.Names = "box_input indi_input"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.GeomInput.box_input.InputType = 'Box'
-LW_Test.GeomInput.box_input.GeomName = 'domain'
+LW_Test.GeomInput.box_input.InputType = "Box"
+LW_Test.GeomInput.box_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_Test.Geom.domain.Lower.X = 0.0
 LW_Test.Geom.domain.Lower.Y = 0.0
@@ -91,15 +95,17 @@
 LW_Test.Geom.domain.Upper.X = 41000.0
 LW_Test.Geom.domain.Upper.Y = 41000.0
 LW_Test.Geom.domain.Upper.Z = 100.0
-LW_Test.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+LW_Test.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Indicator Geometry Input
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.GeomInput.indi_input.InputType = 'IndicatorField'
-LW_Test.GeomInput.indi_input.GeomNames = 's1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 g1 g2 g3 g4 g5 g6 g7 g8'
-LW_Test.Geom.indi_input.FileName = 'IndicatorFile_Gleeson.50z.pfb'
+LW_Test.GeomInput.indi_input.InputType = "IndicatorField"
+LW_Test.GeomInput.indi_input.GeomNames = (
+    "s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 g1 g2 g3 g4 g5 g6 g7 g8"
+)
+LW_Test.Geom.indi_input.FileName = "IndicatorFile_Gleeson.50z.pfb"
 
 LW_Test.GeomInput.s1.Value = 1
 LW_Test.GeomInput.s2.Value = 2
@@ -123,226 +129,226 @@
 LW_Test.GeomInput.g7.Value = 27
 LW_Test.GeomInput.g8.Value = 28
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Permeability (values in m/hr)
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.Geom.Perm.Names = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9 g2 g3 g6 g8'
+LW_Test.Geom.Perm.Names = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9 g2 g3 g6 g8"
 
-LW_Test.Geom.domain.Perm.Type = 'Constant'
+LW_Test.Geom.domain.Perm.Type = "Constant"
 LW_Test.Geom.domain.Perm.Value = 0.2
 
-LW_Test.Geom.s1.Perm.Type = 'Constant'
+LW_Test.Geom.s1.Perm.Type = "Constant"
 LW_Test.Geom.s1.Perm.Value = 0.269022595
 
-LW_Test.Geom.s2.Perm.Type = 'Constant'
+LW_Test.Geom.s2.Perm.Type = "Constant"
 LW_Test.Geom.s2.Perm.Value = 0.043630356
 
-LW_Test.Geom.s3.Perm.Type = 'Constant'
+LW_Test.Geom.s3.Perm.Type = "Constant"
 LW_Test.Geom.s3.Perm.Value = 0.015841225
 
-LW_Test.Geom.s4.Perm.Type = 'Constant'
+LW_Test.Geom.s4.Perm.Type = "Constant"
 LW_Test.Geom.s4.Perm.Value = 0.007582087
 
-LW_Test.Geom.s5.Perm.Type = 'Constant'
+LW_Test.Geom.s5.Perm.Type = "Constant"
 LW_Test.Geom.s5.Perm.Value = 0.01818816
 
-LW_Test.Geom.s6.Perm.Type = 'Constant'
+LW_Test.Geom.s6.Perm.Type = "Constant"
 LW_Test.Geom.s6.Perm.Value = 0.005009435
 
-LW_Test.Geom.s7.Perm.Type = 'Constant'
+LW_Test.Geom.s7.Perm.Type = "Constant"
 LW_Test.Geom.s7.Perm.Value = 0.005492736
 
-LW_Test.Geom.s8.Perm.Type = 'Constant'
+LW_Test.Geom.s8.Perm.Type = "Constant"
 LW_Test.Geom.s8.Perm.Value = 0.004675077
 
-LW_Test.Geom.s9.Perm.Type = 'Constant'
+LW_Test.Geom.s9.Perm.Type = "Constant"
 LW_Test.Geom.s9.Perm.Value = 0.003386794
 
-LW_Test.Geom.g2.Perm.Type = 'Constant'
+LW_Test.Geom.g2.Perm.Type = "Constant"
 LW_Test.Geom.g2.Perm.Value = 0.025
 
-LW_Test.Geom.g3.Perm.Type = 'Constant'
+LW_Test.Geom.g3.Perm.Type = "Constant"
 LW_Test.Geom.g3.Perm.Value = 0.059
 
-LW_Test.Geom.g6.Perm.Type = 'Constant'
+LW_Test.Geom.g6.Perm.Type = "Constant"
 LW_Test.Geom.g6.Perm.Value = 0.2
 
-LW_Test.Geom.g8.Perm.Type = 'Constant'
+LW_Test.Geom.g8.Perm.Type = "Constant"
 LW_Test.Geom.g8.Perm.Value = 0.68
 
-LW_Test.Perm.TensorType = 'TensorByGeom'
-LW_Test.Geom.Perm.TensorByGeom.Names = 'domain'
+LW_Test.Perm.TensorType = "TensorByGeom"
+LW_Test.Geom.Perm.TensorByGeom.Names = "domain"
 LW_Test.Geom.domain.Perm.TensorValX = 1.0
 LW_Test.Geom.domain.Perm.TensorValY = 1.0
 LW_Test.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.SpecificStorage.Type = 'Constant'
-LW_Test.SpecificStorage.GeomNames = 'domain'
+LW_Test.SpecificStorage.Type = "Constant"
+LW_Test.SpecificStorage.GeomNames = "domain"
 LW_Test.Geom.domain.SpecificStorage.Value = 1.0e-5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.Phase.Names = 'water'
-LW_Test.Phase.water.Density.Type = 'Constant'
+LW_Test.Phase.Names = "water"
+LW_Test.Phase.water.Density.Type = "Constant"
 LW_Test.Phase.water.Density.Value = 1.0
-LW_Test.Phase.water.Viscosity.Type = 'Constant'
+LW_Test.Phase.water.Viscosity.Type = "Constant"
 LW_Test.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.Contaminants.Names = ''
+LW_Test.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_Test.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Timing (time units is set by units of permeability)
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_Test.TimingInfo.BaseUnit = 1.0
 LW_Test.TimingInfo.StartCount = 0
 LW_Test.TimingInfo.StartTime = 0.0
 LW_Test.TimingInfo.StopTime = 12.0
 LW_Test.TimingInfo.DumpInterval = 24.0
-LW_Test.TimeStep.Type = 'Constant'
+LW_Test.TimeStep.Type = "Constant"
 LW_Test.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.Geom.Porosity.GeomNames = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9'
+LW_Test.Geom.Porosity.GeomNames = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9"
 
-LW_Test.Geom.domain.Porosity.Type = 'Constant'
+LW_Test.Geom.domain.Porosity.Type = "Constant"
 LW_Test.Geom.domain.Porosity.Value = 0.4
 
-LW_Test.Geom.s1.Porosity.Type = 'Constant'
+LW_Test.Geom.s1.Porosity.Type = "Constant"
 LW_Test.Geom.s1.Porosity.Value = 0.375
 
-LW_Test.Geom.s2.Porosity.Type = 'Constant'
+LW_Test.Geom.s2.Porosity.Type = "Constant"
 LW_Test.Geom.s2.Porosity.Value = 0.39
 
-LW_Test.Geom.s3.Porosity.Type = 'Constant'
+LW_Test.Geom.s3.Porosity.Type = "Constant"
 LW_Test.Geom.s3.Porosity.Value = 0.387
 
-LW_Test.Geom.s4.Porosity.Type = 'Constant'
+LW_Test.Geom.s4.Porosity.Type = "Constant"
 LW_Test.Geom.s4.Porosity.Value = 0.439
 
-LW_Test.Geom.s5.Porosity.Type = 'Constant'
+LW_Test.Geom.s5.Porosity.Type = "Constant"
 LW_Test.Geom.s5.Porosity.Value = 0.489
 
-LW_Test.Geom.s6.Porosity.Type = 'Constant'
+LW_Test.Geom.s6.Porosity.Type = "Constant"
 LW_Test.Geom.s6.Porosity.Value = 0.399
 
-LW_Test.Geom.s7.Porosity.Type = 'Constant'
+LW_Test.Geom.s7.Porosity.Type = "Constant"
 LW_Test.Geom.s7.Porosity.Value = 0.384
 
-LW_Test.Geom.s8.Porosity.Type = 'Constant'
+LW_Test.Geom.s8.Porosity.Type = "Constant"
 LW_Test.Geom.s8.Porosity.Value = 0.482
 
-LW_Test.Geom.s9.Porosity.Type = 'Constant'
+LW_Test.Geom.s9.Porosity.Type = "Constant"
 LW_Test.Geom.s9.Porosity.Value = 0.442
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.Domain.GeomName = 'domain'
+LW_Test.Domain.GeomName = "domain"
 
-#----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 # Mobility
-#----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 
-LW_Test.Phase.water.Mobility.Type = 'Constant'
+LW_Test.Phase.water.Mobility.Type = "Constant"
 LW_Test.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.Wells.Names = ''
+LW_Test.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.Cycle.Names = 'constant'
-LW_Test.Cycle.constant.Names = 'alltime'
+LW_Test.Cycle.Names = "constant"
+LW_Test.Cycle.constant.Names = "alltime"
 LW_Test.Cycle.constant.alltime.Length = 1
 LW_Test.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 LW_Test.BCPressure.PatchNames = LW_Test.Geom.domain.Patches
 
-LW_Test.Patch.x_lower.BCPressure.Type = 'FluxConst'
-LW_Test.Patch.x_lower.BCPressure.Cycle = 'constant'
+LW_Test.Patch.x_lower.BCPressure.Type = "FluxConst"
+LW_Test.Patch.x_lower.BCPressure.Cycle = "constant"
 LW_Test.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-LW_Test.Patch.y_lower.BCPressure.Type = 'FluxConst'
-LW_Test.Patch.y_lower.BCPressure.Cycle = 'constant'
+LW_Test.Patch.y_lower.BCPressure.Type = "FluxConst"
+LW_Test.Patch.y_lower.BCPressure.Cycle = "constant"
 LW_Test.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-LW_Test.Patch.z_lower.BCPressure.Type = 'FluxConst'
-LW_Test.Patch.z_lower.BCPressure.Cycle = 'constant'
+LW_Test.Patch.z_lower.BCPressure.Type = "FluxConst"
+LW_Test.Patch.z_lower.BCPressure.Cycle = "constant"
 LW_Test.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-LW_Test.Patch.x_upper.BCPressure.Type = 'FluxConst'
-LW_Test.Patch.x_upper.BCPressure.Cycle = 'constant'
+LW_Test.Patch.x_upper.BCPressure.Type = "FluxConst"
+LW_Test.Patch.x_upper.BCPressure.Cycle = "constant"
 LW_Test.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-LW_Test.Patch.y_upper.BCPressure.Type = 'FluxConst'
-LW_Test.Patch.y_upper.BCPressure.Cycle = 'constant'
+LW_Test.Patch.y_upper.BCPressure.Type = "FluxConst"
+LW_Test.Patch.y_upper.BCPressure.Cycle = "constant"
 LW_Test.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-LW_Test.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-LW_Test.Patch.z_upper.BCPressure.Cycle = 'constant'
+LW_Test.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+LW_Test.Patch.z_upper.BCPressure.Cycle = "constant"
 LW_Test.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Topo slopes in x-direction
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.TopoSlopesX.Type = 'PFBFile'
-LW_Test.TopoSlopesX.GeomNames = 'domain'
-LW_Test.TopoSlopesX.FileName = 'LW.slopex.pfb'
+LW_Test.TopoSlopesX.Type = "PFBFile"
+LW_Test.TopoSlopesX.GeomNames = "domain"
+LW_Test.TopoSlopesX.FileName = "LW.slopex.pfb"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Topo slopes in y-direction
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.TopoSlopesY.Type = 'PFBFile'
-LW_Test.TopoSlopesY.GeomNames = 'domain'
-LW_Test.TopoSlopesY.FileName = 'LW.slopey.pfb'
+LW_Test.TopoSlopesY.Type = "PFBFile"
+LW_Test.TopoSlopesY.GeomNames = "domain"
+LW_Test.TopoSlopesY.FileName = "LW.slopey.pfb"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mannings coefficient
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.Mannings.Type = 'Constant'
-LW_Test.Mannings.GeomNames = 'domain'
+LW_Test.Mannings.Type = "Constant"
+LW_Test.Mannings.GeomNames = "domain"
 LW_Test.Mannings.Geom.domain.Value = 5.52e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.Phase.RelPerm.Type = 'VanGenuchten'
-LW_Test.Phase.RelPerm.GeomNames = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9 '
+LW_Test.Phase.RelPerm.Type = "VanGenuchten"
+LW_Test.Phase.RelPerm.GeomNames = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9 "
 
 LW_Test.Geom.domain.RelPerm.Alpha = 3.5
 LW_Test.Geom.domain.RelPerm.N = 2.0
@@ -374,15 +380,15 @@
 LW_Test.Geom.s9.RelPerm.Alpha = 1.585
 LW_Test.Geom.s9.RelPerm.N = 2.413
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Saturation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.Phase.Saturation.Type = 'VanGenuchten'
-LW_Test.Phase.Saturation.GeomNames = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9 '
+LW_Test.Phase.Saturation.Type = "VanGenuchten"
+LW_Test.Phase.Saturation.GeomNames = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9 "
 
 LW_Test.Geom.domain.Saturation.Alpha = 3.5
-LW_Test.Geom.domain.Saturation.N = 2.
+LW_Test.Geom.domain.Saturation.N = 2.0
 LW_Test.Geom.domain.Saturation.SRes = 0.2
 LW_Test.Geom.domain.Saturation.SSat = 1.0
 
@@ -431,50 +437,50 @@
 LW_Test.Geom.s9.Saturation.SRes = 0.000001
 LW_Test.Geom.s9.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.PhaseSources.water.Type = 'Constant'
-LW_Test.PhaseSources.water.GeomNames = 'domain'
+LW_Test.PhaseSources.water.Type = "Constant"
+LW_Test.PhaseSources.water.GeomNames = "domain"
 LW_Test.PhaseSources.water.Geom.domain.Value = 0.0
 
-#----------------------------------------------------------------
+# ----------------------------------------------------------------
 # CLM Settings:
 # ------------------------------------------------------------
 
-LW_Test.Solver.LSM = 'CLM'
-LW_Test.Solver.CLM.CLMFileDir = '.'
+LW_Test.Solver.LSM = "CLM"
+LW_Test.Solver.CLM.CLMFileDir = "."
 LW_Test.Solver.CLM.Print1dOut = False
 LW_Test.Solver.CLM.CLMDumpInterval = 1
 
-LW_Test.Solver.CLM.MetForcing = '3D'
-LW_Test.Solver.CLM.MetFileName = 'NLDAS'
-LW_Test.Solver.CLM.MetFilePath = '.'
+LW_Test.Solver.CLM.MetForcing = "3D"
+LW_Test.Solver.CLM.MetFileName = "NLDAS"
+LW_Test.Solver.CLM.MetFilePath = "."
 LW_Test.Solver.CLM.MetFileNT = 24
 LW_Test.Solver.CLM.IstepStart = 1
 
-LW_Test.Solver.CLM.EvapBeta = 'Linear'
-LW_Test.Solver.CLM.VegWaterStress = 'Saturation'
+LW_Test.Solver.CLM.EvapBeta = "Linear"
+LW_Test.Solver.CLM.VegWaterStress = "Saturation"
 LW_Test.Solver.CLM.ResSat = 0.1
 LW_Test.Solver.CLM.WiltingPoint = 0.12
 LW_Test.Solver.CLM.FieldCapacity = 0.98
-LW_Test.Solver.CLM.IrrigationType = 'none'
+LW_Test.Solver.CLM.IrrigationType = "none"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-LW_Test.ICPressure.Type = 'PFBFile'
-LW_Test.ICPressure.GeomNames = 'domain'
-LW_Test.Geom.domain.ICPressure.RefPatch = 'z_upper'
-LW_Test.Geom.domain.ICPressure.FileName = 'press.init.pfb'
+LW_Test.ICPressure.Type = "PFBFile"
+LW_Test.ICPressure.GeomNames = "domain"
+LW_Test.Geom.domain.ICPressure.RefPatch = "z_upper"
+LW_Test.Geom.domain.ICPressure.FileName = "press.init.pfb"
 
-#-------------------------------------------------------------
+# -------------------------------------------------------------
 # Outputs
 # ------------------------------------------------------------
 
-#Writing output (all pfb):
+# Writing output (all pfb):
 LW_Test.Solver.PrintSubsurfData = False
 LW_Test.Solver.PrintPressure = True
 LW_Test.Solver.PrintSaturation = True
@@ -494,69 +500,85 @@
 LW_Test.Solver.WriteSiloOverlandSum = False
 LW_Test.Solver.WriteSiloCLM = False
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-LW_Test.KnownSolution = 'NoKnownSolution'
+LW_Test.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 # ParFlow Solution
-LW_Test.Solver = 'Richards'
+LW_Test.Solver = "Richards"
 LW_Test.Solver.TerrainFollowingGrid = True
 LW_Test.Solver.Nonlinear.VariableDz = False
 
 LW_Test.Solver.MaxIter = 25000
-LW_Test.Solver.Drop = 1E-20
-LW_Test.Solver.AbsTol = 1E-8
+LW_Test.Solver.Drop = 1e-20
+LW_Test.Solver.AbsTol = 1e-8
 LW_Test.Solver.MaxConvergenceFailures = 8
 LW_Test.Solver.Nonlinear.MaxIter = 80
 LW_Test.Solver.Nonlinear.ResidualTol = 1e-6
 
 ## new solver settings for Terrain Following Grid
-LW_Test.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+LW_Test.Solver.Nonlinear.EtaChoice = "EtaConstant"
 LW_Test.Solver.Nonlinear.EtaValue = 0.001
 LW_Test.Solver.Nonlinear.UseJacobian = True
 LW_Test.Solver.Nonlinear.DerivativeEpsilon = 1e-16
 LW_Test.Solver.Nonlinear.StepTol = 1e-30
-LW_Test.Solver.Nonlinear.Globalization = 'LineSearch'
+LW_Test.Solver.Nonlinear.Globalization = "LineSearch"
 LW_Test.Solver.Linear.KrylovDimension = 70
 LW_Test.Solver.Linear.MaxRestarts = 2
 
-LW_Test.Solver.Linear.Preconditioner = 'PFMGOctree'
-LW_Test.Solver.Linear.Preconditioner.PCMatrixType = 'FullJacobian'
+LW_Test.Solver.Linear.Preconditioner = "PFMGOctree"
+LW_Test.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Distribute CLM and ParFlow inputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-#LW_Test.dist(dir_name + '/' + 'drv_clmin.dat')
-#LW_Test.dist(dir_name + '/' + 'drv_vegm.alluv.dat')
-#LW_Test.dist(dir_name + '/' + 'drv_vegp.dat')
-LW_Test.dist(dir_name + '/' + 'LW.slopex.pfb')
-LW_Test.dist(dir_name + '/' + 'LW.slopey.pfb')
-LW_Test.dist(dir_name + '/' + 'IndicatorFile_Gleeson.50z.pfb')
-LW_Test.dist(dir_name + '/' + 'press.init.pfb')
+# LW_Test.dist(dir_name + '/' + 'drv_clmin.dat')
+# LW_Test.dist(dir_name + '/' + 'drv_vegm.alluv.dat')
+# LW_Test.dist(dir_name + '/' + 'drv_vegp.dat')
+LW_Test.dist(dir_name + "/" + "LW.slopex.pfb")
+LW_Test.dist(dir_name + "/" + "LW.slopey.pfb")
+LW_Test.dist(dir_name + "/" + "IndicatorFile_Gleeson.50z.pfb")
+LW_Test.dist(dir_name + "/" + "press.init.pfb")
 
 nldas_files = [
-  'NLDAS.DSWR.000001_to_000024.pfb',
-  'NLDAS.DLWR.000001_to_000024.pfb',
-  'NLDAS.APCP.000001_to_000024.pfb',
-  'NLDAS.Temp.000001_to_000024.pfb',
-  'NLDAS.UGRD.000001_to_000024.pfb',
-  'NLDAS.VGRD.000001_to_000024.pfb',
-  'NLDAS.Press.000001_to_000024.pfb',
-  'NLDAS.SPFH.000001_to_000024.pfb'
-  ]
+    "NLDAS.DSWR.000001_to_000024.pfb",
+    "NLDAS.DLWR.000001_to_000024.pfb",
+    "NLDAS.APCP.000001_to_000024.pfb",
+    "NLDAS.Temp.000001_to_000024.pfb",
+    "NLDAS.UGRD.000001_to_000024.pfb",
+    "NLDAS.VGRD.000001_to_000024.pfb",
+    "NLDAS.Press.000001_to_000024.pfb",
+    "NLDAS.SPFH.000001_to_000024.pfb",
+]
 
 for file in nldas_files:
-  cp('$PF_SRC/test/tcl/washita/NLDAS/' + file, dir_name)
-  LW_Test.dist(dir_name + '/' + file)
-#-----------------------------------------------------------------------------
+    cp("$PF_SRC/test/tcl/washita/NLDAS/" + file, dir_name)
+    LW_Test.dist(dir_name + "/" + file)
+# -----------------------------------------------------------------------------
 # Run Simulation
-#-----------------------------------------------------------------------------
-
+# -----------------------------------------------------------------------------
+
+# Corrupt the indicator file and run check_nans()
+corrupted_indicator = read_pfb(os.path.join(dir_name, "IndicatorFile_Gleeson.50z.pfb"))
+corrupted_indicator[0, 0, 0] = np.nan
+write_pfb(os.path.join(dir_name, "corrupted_indicator.pfb"), corrupted_indicator)
+LW_Test.Geom.indi_input.FileName = "corrupted_indicator.pfb"
+try:
+    LW_Test.check_nans(working_directory=dir_name)
+    raise AssertionError("ValueError was not raised for corrupted indicator file.")
+except ValueError as e:
+    assert "corrupted_indicator.pfb" in str(e)
+
+# Restore the original
+LW_Test.Geom.indi_input.FileName = "IndicatorFile_Gleeson.50z.pfb"
+
+# Check NaNs for original inputs
+LW_Test.check_nans(working_directory=dir_name)
 LW_Test.run(working_directory=dir_name)
diff --git a/test/python/washita/LW_Timing.py b/test/python/washita/LW_Timing.py
index f3052ec73..ccd8f9639 100644
--- a/test/python/washita/LW_Timing.py
+++ b/test/python/washita/LW_Timing.py
@@ -7,25 +7,25 @@
 # package   require parflow
 LW_Timing = Run("LW_Timing", __file__)
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Make a directory for the simulation run, files will be copied to this
 # directory for running.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # file mkdir "Outputs"
 # cd "./Outputs"
 
-LW_Timing. = 'FileVersion 4'
+LW_Timing.FileVersion = "FileVersion 4"
 
-#-----------------------------------------------------------------------------
-# Set Processor topology 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+# Set Processor topology
+# -----------------------------------------------------------------------------
 LW_Timing.Process.Topology.P = 1
 LW_Timing.Process.Topology.Q = 1
 LW_Timing.Process.Topology.R = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Computational Grid
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 LW_Timing.ComputationalGrid.Lower.X = 0.0
 LW_Timing.ComputationalGrid.Lower.Y = 0.0
 LW_Timing.ComputationalGrid.Lower.Z = 0.0
@@ -38,35 +38,37 @@
 LW_Timing.ComputationalGrid.NY = 41
 LW_Timing.ComputationalGrid.NZ = 50
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Names of the GeomInputs
-#-----------------------------------------------------------------------------
-LW_Timing.GeomInput.Names = 'box_input indi_input'
+# -----------------------------------------------------------------------------
+LW_Timing.GeomInput.Names = "box_input indi_input"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain Geometry Input
-#-----------------------------------------------------------------------------
-LW_Timing.GeomInput.box_input.InputType = 'Box'
-LW_Timing.GeomInput.box_input.GeomName = 'domain'
+# -----------------------------------------------------------------------------
+LW_Timing.GeomInput.box_input.InputType = "Box"
+LW_Timing.GeomInput.box_input.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
-# Domain Geometry 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+# Domain Geometry
+# -----------------------------------------------------------------------------
 LW_Timing.Geom.domain.Lower.X = 0.0
 LW_Timing.Geom.domain.Lower.Y = 0.0
 LW_Timing.Geom.domain.Lower.Z = 0.0
-#  
+#
 LW_Timing.Geom.domain.Upper.X = 41000.0
 LW_Timing.Geom.domain.Upper.Y = 41000.0
 LW_Timing.Geom.domain.Upper.Z = 100.0
-LW_Timing.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+LW_Timing.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Indicator Geometry Input
-#-----------------------------------------------------------------------------
-LW_Timing.GeomInput.indi_input.InputType = 'IndicatorField'
-LW_Timing.GeomInput.indi_input.GeomNames = 's1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 g1 g2 g3 g4 g5 g6 g7 g8'
-LW_Timing.Geom.indi_input.FileName = 'IndicatorFile_Gleeson.50z.pfb'
+# -----------------------------------------------------------------------------
+LW_Timing.GeomInput.indi_input.InputType = "IndicatorField"
+LW_Timing.GeomInput.indi_input.GeomNames = (
+    "s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 g1 g2 g3 g4 g5 g6 g7 g8"
+)
+LW_Timing.Geom.indi_input.FileName = "IndicatorFile_Gleeson.50z.pfb"
 
 LW_Timing.GeomInput.s1.Value = 1
 LW_Timing.GeomInput.s2.Value = 2
@@ -90,210 +92,210 @@
 LW_Timing.GeomInput.g7.Value = 27
 LW_Timing.GeomInput.g8.Value = 28
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Permeability (values in m/hr)
-#-----------------------------------------------------------------------------
-LW_Timing.Geom.Perm.Names = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9 g2 g3 g6 g8'
+# -----------------------------------------------------------------------------
+LW_Timing.Geom.Perm.Names = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9 g2 g3 g6 g8"
 
-LW_Timing.Geom.domain.Perm.Type = 'Constant'
+LW_Timing.Geom.domain.Perm.Type = "Constant"
 LW_Timing.Geom.domain.Perm.Value = 0.2
 
-LW_Timing.Geom.s1.Perm.Type = 'Constant'
+LW_Timing.Geom.s1.Perm.Type = "Constant"
 LW_Timing.Geom.s1.Perm.Value = 0.269022595
 
-LW_Timing.Geom.s2.Perm.Type = 'Constant'
+LW_Timing.Geom.s2.Perm.Type = "Constant"
 LW_Timing.Geom.s2.Perm.Value = 0.043630356
 
-LW_Timing.Geom.s3.Perm.Type = 'Constant'
+LW_Timing.Geom.s3.Perm.Type = "Constant"
 LW_Timing.Geom.s3.Perm.Value = 0.015841225
 
-LW_Timing.Geom.s4.Perm.Type = 'Constant'
+LW_Timing.Geom.s4.Perm.Type = "Constant"
 LW_Timing.Geom.s4.Perm.Value = 0.007582087
 
-LW_Timing.Geom.s5.Perm.Type = 'Constant'
+LW_Timing.Geom.s5.Perm.Type = "Constant"
 LW_Timing.Geom.s5.Perm.Value = 0.01818816
 
-LW_Timing.Geom.s6.Perm.Type = 'Constant'
+LW_Timing.Geom.s6.Perm.Type = "Constant"
 LW_Timing.Geom.s6.Perm.Value = 0.005009435
 
-LW_Timing.Geom.s7.Perm.Type = 'Constant'
+LW_Timing.Geom.s7.Perm.Type = "Constant"
 LW_Timing.Geom.s7.Perm.Value = 0.005492736
 
-LW_Timing.Geom.s8.Perm.Type = 'Constant'
+LW_Timing.Geom.s8.Perm.Type = "Constant"
 LW_Timing.Geom.s8.Perm.Value = 0.004675077
 
-LW_Timing.Geom.s9.Perm.Type = 'Constant'
+LW_Timing.Geom.s9.Perm.Type = "Constant"
 LW_Timing.Geom.s9.Perm.Value = 0.003386794
 
-LW_Timing.Geom.g2.Perm.Type = 'Constant'
+LW_Timing.Geom.g2.Perm.Type = "Constant"
 LW_Timing.Geom.g2.Perm.Value = 0.025
 
-LW_Timing.Geom.g3.Perm.Type = 'Constant'
+LW_Timing.Geom.g3.Perm.Type = "Constant"
 LW_Timing.Geom.g3.Perm.Value = 0.059
 
-LW_Timing.Geom.g6.Perm.Type = 'Constant'
+LW_Timing.Geom.g6.Perm.Type = "Constant"
 LW_Timing.Geom.g6.Perm.Value = 0.2
 
-LW_Timing.Geom.g8.Perm.Type = 'Constant'
+LW_Timing.Geom.g8.Perm.Type = "Constant"
 LW_Timing.Geom.g8.Perm.Value = 0.68
 
-LW_Timing.Perm.TensorType = 'TensorByGeom'
-LW_Timing.Geom.Perm.TensorByGeom.Names = 'domain'
-LW_Timing.Geom.domain.Perm.TensorValX = 1.0d0
-LW_Timing.Geom.domain.Perm.TensorValY = 1.0d0
-LW_Timing.Geom.domain.Perm.TensorValZ = 1.0d0
+LW_Timing.Perm.TensorType = "TensorByGeom"
+LW_Timing.Geom.Perm.TensorByGeom.Names = "domain"
+LW_Timing.Geom.domain.Perm.TensorValX = 1.0
+LW_Timing.Geom.domain.Perm.TensorValY = 1.0
+LW_Timing.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
-LW_Timing.SpecificStorage.Type = 'Constant'
-LW_Timing.SpecificStorage.GeomNames = 'domain'
+# -----------------------------------------------------------------------------
+LW_Timing.SpecificStorage.Type = "Constant"
+LW_Timing.SpecificStorage.GeomNames = "domain"
 LW_Timing.Geom.domain.SpecificStorage.Value = 1.0e-5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
-LW_Timing.Phase.Names = 'water'
-LW_Timing.Phase.water.Density.Type = 'Constant'
+# -----------------------------------------------------------------------------
+LW_Timing.Phase.Names = "water"
+LW_Timing.Phase.water.Density.Type = "Constant"
 LW_Timing.Phase.water.Density.Value = 1.0
-LW_Timing.Phase.water.Viscosity.Type = 'Constant'
+LW_Timing.Phase.water.Viscosity.Type = "Constant"
 LW_Timing.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
-LW_Timing.Contaminants.Names = ''
+# -----------------------------------------------------------------------------
+LW_Timing.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 LW_Timing.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Timing (time units is set by units of permeability)
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 LW_Timing.TimingInfo.BaseUnit = 1.0
 LW_Timing.TimingInfo.StartCount = 0.0
 LW_Timing.TimingInfo.StartTime = 0.0
 LW_Timing.TimingInfo.StopTime = 100.0
 LW_Timing.TimingInfo.DumpInterval = 10000.0
-LW_Timing.TimeStep.Type = 'Constant'
+LW_Timing.TimeStep.Type = "Constant"
 LW_Timing.TimeStep.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
-LW_Timing.Geom.Porosity.GeomNames = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9'
+# -----------------------------------------------------------------------------
+LW_Timing.Geom.Porosity.GeomNames = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9"
 
-LW_Timing.Geom.domain.Porosity.Type = 'Constant'
+LW_Timing.Geom.domain.Porosity.Type = "Constant"
 LW_Timing.Geom.domain.Porosity.Value = 0.4
 
-LW_Timing.Geom.s1.Porosity.Type = 'Constant'
+LW_Timing.Geom.s1.Porosity.Type = "Constant"
 LW_Timing.Geom.s1.Porosity.Value = 0.375
 
-LW_Timing.Geom.s2.Porosity.Type = 'Constant'
+LW_Timing.Geom.s2.Porosity.Type = "Constant"
 LW_Timing.Geom.s2.Porosity.Value = 0.39
 
-LW_Timing.Geom.s3.Porosity.Type = 'Constant'
+LW_Timing.Geom.s3.Porosity.Type = "Constant"
 LW_Timing.Geom.s3.Porosity.Value = 0.387
 
-LW_Timing.Geom.s4.Porosity.Type = 'Constant'
+LW_Timing.Geom.s4.Porosity.Type = "Constant"
 LW_Timing.Geom.s4.Porosity.Value = 0.439
 
-LW_Timing.Geom.s5.Porosity.Type = 'Constant'
+LW_Timing.Geom.s5.Porosity.Type = "Constant"
 LW_Timing.Geom.s5.Porosity.Value = 0.489
 
-LW_Timing.Geom.s6.Porosity.Type = 'Constant'
+LW_Timing.Geom.s6.Porosity.Type = "Constant"
 LW_Timing.Geom.s6.Porosity.Value = 0.399
 
-LW_Timing.Geom.s7.Porosity.Type = 'Constant'
+LW_Timing.Geom.s7.Porosity.Type = "Constant"
 LW_Timing.Geom.s7.Porosity.Value = 0.384
 
-LW_Timing.Geom.s8.Porosity.Type = 'Constant'
+LW_Timing.Geom.s8.Porosity.Type = "Constant"
 LW_Timing.Geom.s8.Porosity.Value = 0.482
 
-LW_Timing.Geom.s9.Porosity.Type = 'Constant'
+LW_Timing.Geom.s9.Porosity.Type = "Constant"
 LW_Timing.Geom.s9.Porosity.Value = 0.442
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
-LW_Timing.Domain.GeomName = 'domain'
+# -----------------------------------------------------------------------------
+LW_Timing.Domain.GeomName = "domain"
 
-#----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 # Mobility
-#----------------------------------------------------------------------------
-LW_Timing.Phase.water.Mobility.Type = 'Constant'
+# ----------------------------------------------------------------------------
+LW_Timing.Phase.water.Mobility.Type = "Constant"
 LW_Timing.Phase.water.Mobility.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
-LW_Timing.Wells.Names = ''
+# -----------------------------------------------------------------------------
+LW_Timing.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
-LW_Timing.Cycle.Names = 'constant'
-LW_Timing.Cycle.constant.Names = 'alltime'
+# -----------------------------------------------------------------------------
+LW_Timing.Cycle.Names = "constant"
+LW_Timing.Cycle.constant.Names = "alltime"
 LW_Timing.Cycle.constant.alltime.Length = 1
 LW_Timing.Cycle.constant.Repeat = -1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions
-#-----------------------------------------------------------------------------
-LW_Timing.BCPressure.PatchNames = [pfget Geom.domain.Patches]
+# -----------------------------------------------------------------------------
+LW_Timing.BCPressure.PatchNames = LW_Timing.Geom.domain.Patches
 
-LW_Timing.Patch.x_lower.BCPressure.Type = 'FluxConst'
-LW_Timing.Patch.x_lower.BCPressure.Cycle = 'constant'
+LW_Timing.Patch.x_lower.BCPressure.Type = "FluxConst"
+LW_Timing.Patch.x_lower.BCPressure.Cycle = "constant"
 LW_Timing.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-LW_Timing.Patch.y_lower.BCPressure.Type = 'FluxConst'
-LW_Timing.Patch.y_lower.BCPressure.Cycle = 'constant'
+LW_Timing.Patch.y_lower.BCPressure.Type = "FluxConst"
+LW_Timing.Patch.y_lower.BCPressure.Cycle = "constant"
 LW_Timing.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-LW_Timing.Patch.z_lower.BCPressure.Type = 'FluxConst'
-LW_Timing.Patch.z_lower.BCPressure.Cycle = 'constant'
+LW_Timing.Patch.z_lower.BCPressure.Type = "FluxConst"
+LW_Timing.Patch.z_lower.BCPressure.Cycle = "constant"
 LW_Timing.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-LW_Timing.Patch.x_upper.BCPressure.Type = 'FluxConst'
-LW_Timing.Patch.x_upper.BCPressure.Cycle = 'constant'
+LW_Timing.Patch.x_upper.BCPressure.Type = "FluxConst"
+LW_Timing.Patch.x_upper.BCPressure.Cycle = "constant"
 LW_Timing.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-LW_Timing.Patch.y_upper.BCPressure.Type = 'FluxConst'
-LW_Timing.Patch.y_upper.BCPressure.Cycle = 'constant'
+LW_Timing.Patch.y_upper.BCPressure.Type = "FluxConst"
+LW_Timing.Patch.y_upper.BCPressure.Cycle = "constant"
 LW_Timing.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-LW_Timing.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-LW_Timing.Patch.z_upper.BCPressure.Cycle = 'constant'
+LW_Timing.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+LW_Timing.Patch.z_upper.BCPressure.Cycle = "constant"
 LW_Timing.Patch.z_upper.BCPressure.alltime.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Topo slopes in x-direction
-#-----------------------------------------------------------------------------
-LW_Timing.TopoSlopesX.Type = 'PFBFile'
-LW_Timing.TopoSlopesX.GeomNames = 'domain'
-LW_Timing.TopoSlopesX.FileName = 'LW.slopex.pfb'
+# -----------------------------------------------------------------------------
+LW_Timing.TopoSlopesX.Type = "PFBFile"
+LW_Timing.TopoSlopesX.GeomNames = "domain"
+LW_Timing.TopoSlopesX.FileName = "LW.slopex.pfb"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Topo slopes in y-direction
-#-----------------------------------------------------------------------------
-LW_Timing.TopoSlopesY.Type = 'PFBFile'
-LW_Timing.TopoSlopesY.GeomNames = 'domain'
-LW_Timing.TopoSlopesY.FileName = 'LW.slopey.pfb'
+# -----------------------------------------------------------------------------
+LW_Timing.TopoSlopesY.Type = "PFBFile"
+LW_Timing.TopoSlopesY.GeomNames = "domain"
+LW_Timing.TopoSlopesY.FileName = "LW.slopey.pfb"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Mannings coefficient
-#-----------------------------------------------------------------------------
-LW_Timing.Mannings.Type = 'Constant'
-LW_Timing.Mannings.GeomNames = 'domain'
+# -----------------------------------------------------------------------------
+LW_Timing.Mannings.Type = "Constant"
+LW_Timing.Mannings.GeomNames = "domain"
 LW_Timing.Mannings.Geom.domain.Value = 5.52e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
-LW_Timing.Phase.RelPerm.Type = 'VanGenuchten'
-LW_Timing.Phase.RelPerm.GeomNames = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9 '
+# -----------------------------------------------------------------------------
+LW_Timing.Phase.RelPerm.Type = "VanGenuchten"
+LW_Timing.Phase.RelPerm.GeomNames = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9 "
 
 LW_Timing.Geom.domain.RelPerm.Alpha = 3.5
 LW_Timing.Geom.domain.RelPerm.N = 2.0
@@ -325,14 +327,14 @@
 LW_Timing.Geom.s9.RelPerm.Alpha = 1.585
 LW_Timing.Geom.s9.RelPerm.N = 2.413
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Saturation
-#-----------------------------------------------------------------------------
-LW_Timing.Phase.Saturation.Type = 'VanGenuchten'
-LW_Timing.Phase.Saturation.GeomNames = 'domain s1 s2 s3 s4 s5 s6 s7 s8 s9 '
+# -----------------------------------------------------------------------------
+LW_Timing.Phase.Saturation.Type = "VanGenuchten"
+LW_Timing.Phase.Saturation.GeomNames = "domain s1 s2 s3 s4 s5 s6 s7 s8 s9 "
 
 LW_Timing.Geom.domain.Saturation.Alpha = 3.5
-LW_Timing.Geom.domain.Saturation.N = 2.
+LW_Timing.Geom.domain.Saturation.N = 2.0
 LW_Timing.Geom.domain.Saturation.SRes = 0.2
 LW_Timing.Geom.domain.Saturation.SSat = 1.0
 
@@ -381,47 +383,47 @@
 LW_Timing.Geom.s9.Saturation.SRes = 0.000001
 LW_Timing.Geom.s9.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
-LW_Timing.PhaseSources.water.Type = 'Constant'
-LW_Timing.PhaseSources.water.GeomNames = 'domain'
+# -----------------------------------------------------------------------------
+LW_Timing.PhaseSources.water.Type = "Constant"
+LW_Timing.PhaseSources.water.GeomNames = "domain"
 LW_Timing.PhaseSources.water.Geom.domain.Value = 0.0
 
-#----------------------------------------------------------------
+# ----------------------------------------------------------------
 # CLM Settings:
 # ------------------------------------------------------------
-LW_Timing.Solver.LSM = 'CLM'
-LW_Timing.Solver.CLM.CLMFileDir = 'clm_output/'
+LW_Timing.Solver.LSM = "CLM"
+LW_Timing.Solver.CLM.CLMFileDir = "clm_output/"
 LW_Timing.Solver.CLM.Print1dOut = False
 LW_Timing.Solver.BinaryOutDir = False
 LW_Timing.Solver.CLM.CLMDumpInterval = 1000000
 
-LW_Timing.Solver.CLM.MetForcing = 3D
-LW_Timing.Solver.CLM.MetFileName = 'NLDAS'
-LW_Timing.Solver.CLM.MetFilePath = '.'
+LW_Timing.Solver.CLM.MetForcing = "3D"
+LW_Timing.Solver.CLM.MetFileName = "NLDAS"
+LW_Timing.Solver.CLM.MetFilePath = "."
 LW_Timing.Solver.CLM.MetFileNT = 24
 LW_Timing.Solver.CLM.IstepStart = 1
 
-LW_Timing.Solver.CLM.EvapBeta = 'Linear'
-LW_Timing.Solver.CLM.VegWaterStress = 'Saturation'
+LW_Timing.Solver.CLM.EvapBeta = "Linear"
+LW_Timing.Solver.CLM.VegWaterStress = "Saturation"
 LW_Timing.Solver.CLM.ResSat = 0.1
 LW_Timing.Solver.CLM.WiltingPoint = 0.12
 LW_Timing.Solver.CLM.FieldCapacity = 0.98
-LW_Timing.Solver.CLM.IrrigationType = 'none'
+LW_Timing.Solver.CLM.IrrigationType = "none"
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
-LW_Timing.ICPressure.Type = 'PFBFile'
-LW_Timing.ICPressure.GeomNames = 'domain'
-LW_Timing.Geom.domain.ICPressure.RefPatch = 'z_upper'
-LW_Timing.Geom.domain.ICPressure.FileName = 'press.init.pfb'
+# ---------------------------------------------------------
+LW_Timing.ICPressure.Type = "PFBFile"
+LW_Timing.ICPressure.GeomNames = "domain"
+LW_Timing.Geom.domain.ICPressure.RefPatch = "z_upper"
+LW_Timing.Geom.domain.ICPressure.FileName = "press.init.pfb"
 
-#----------------------------------------------------------------
+# ----------------------------------------------------------------
 # Outputs
 # ------------------------------------------------------------
-#Writing output (all pfb):
+# Writing output (all pfb):
 LW_Timing.Solver.PrintSubsurfData = False
 LW_Timing.Solver.PrintPressure = True
 LW_Timing.Solver.PrintSaturation = True
@@ -442,43 +444,43 @@
 LW_Timing.Solver.WriteSiloCLM = False
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
-LW_Timing.KnownSolution = 'NoKnownSolution'
+# -----------------------------------------------------------------------------
+LW_Timing.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # ParFlow Solution
-LW_Timing.Solver = 'Richards'
+LW_Timing.Solver = "Richards"
 LW_Timing.Solver.TerrainFollowingGrid = True
 LW_Timing.Solver.Nonlinear.VariableDz = False
 
 LW_Timing.Solver.MaxIter = 25000
-LW_Timing.Solver.Drop = 1E-20
-LW_Timing.Solver.AbsTol = 1E-8
+LW_Timing.Solver.Drop = 1e-20
+LW_Timing.Solver.AbsTol = 1e-8
 LW_Timing.Solver.MaxConvergenceFailures = 8
 LW_Timing.Solver.Nonlinear.MaxIter = 80
 LW_Timing.Solver.Nonlinear.ResidualTol = 1e-6
 
 ## new solver settings for Terrain Following Grid
-LW_Timing.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+LW_Timing.Solver.Nonlinear.EtaChoice = "EtaConstant"
 LW_Timing.Solver.Nonlinear.EtaValue = 0.001
 LW_Timing.Solver.Nonlinear.UseJacobian = True
 LW_Timing.Solver.Nonlinear.DerivativeEpsilon = 1e-16
 LW_Timing.Solver.Nonlinear.StepTol = 1e-30
-LW_Timing.Solver.Nonlinear.Globalization = 'LineSearch'
+LW_Timing.Solver.Nonlinear.Globalization = "LineSearch"
 LW_Timing.Solver.Linear.KrylovDimension = 70
 LW_Timing.Solver.Linear.MaxRestarts = 2
 
-LW_Timing.Solver.Linear.Preconditioner = 'PFMGOctree'
-LW_Timing.Solver.Linear.Preconditioner.PCMatrixType = 'FullJacobian'
+LW_Timing.Solver.Linear.Preconditioner = "PFMGOctree"
+LW_Timing.Solver.Linear.Preconditioner.PCMatrixType = "FullJacobian"
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Copy files and distribute.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 # ParFlow Inputs
 path = "../../parflow_input"
@@ -486,16 +488,16 @@
 #     file copy -force [format "%s/%s.pfb" $path $file] .
 # }
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Distribute inputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # pfdist -nz 1 LW.slopex.pfb
 # pfdist -nz 1 LW.slopey.pfb
 
 # pfdist IndicatorFile_Gleeson.50z.pfb
 # pfdist press.init.pfb
 
-#CLM Inputs
+# CLM Inputs
 path = "../../clm_input"
 # foreach file "drv_clmin drv_vegp drv_vegm.alluv" {
 #     file copy -force [format "%s/%s.dat" $path $file] .
@@ -503,15 +505,15 @@
 
 path = "../../NLDAS"
 
-files = [glob "$path/NLDAS.DSWR.*.pfb"]
+# files = [glob "$path/NLDAS.DSWR.*.pfb"]
 
 time_periods = []
 # foreach file $files {
-#     regexp {NLDAS\.DSWR\.(.*)\.pfb} $file full time 
+#     regexp {NLDAS\.DSWR\.(.*)\.pfb} $file full time
 #     lappend time_periods $time
 # }
 
-NldasVariables = [list "DSWR" "DLWR" "APCP" "Temp" "UGRD" "VGRD" "Press" "SPFH"]
+# NldasVariables = [list "DSWR" "DLWR" "APCP" "Temp" "UGRD" "VGRD" "Press" "SPFH"]
 
 # foreach time_period $time_periods {
 #     foreach variable $NldasVariables {
@@ -524,33 +526,33 @@
 # file delete correct_output
 # file link -symbolic correct_output "../correct_output"
 
-#-----------------------------------------------------------------------------
-# Run Simulation 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+# Run Simulation
+# -----------------------------------------------------------------------------
 # set runname "LW"
 # pfrun    $runname
 
 # puts "ParFlow run Complete"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Undistribute outputs
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # pfundist $runname
 
-StartTime = [expr int([pfget TimingInfo.StartTime])]
-StopTime = [expr int([pfget TimingInfo.StopTime])]
+# StartTime = [expr int([pfget TimingInfo.StartTime])]
+# StopTime = [expr int([pfget TimingInfo.StopTime])]
 
-ClmVariables = [list "eflx_lh_tot" "qflx_evap_soi" "swe_out" "eflx_lwrad_out" "qflx_evap_tot" "t_grnd" "eflx_sh_tot" "qflx_evap_veg" "t_soil" "eflx_soil_grnd" "qflx_infl" "qflx_evap_grnd" "qflx_tran_veg" ]
-# for {set i $StartTime} { $i <= $StopTime } {incr i} { 
+# ClmVariables = [list "eflx_lh_tot" "qflx_evap_soi" "swe_out" "eflx_lwrad_out" "qflx_evap_tot" "t_grnd" "eflx_sh_tot" "qflx_evap_veg" "t_soil" "eflx_soil_grnd" "qflx_infl" "qflx_evap_grnd" "qflx_tran_veg" ]
+# for {set i $StartTime} { $i <= $StopTime } {incr i} {
 #     set step [format "%05d" $i]
 #     foreach variable $ClmVariables {
 #         pfundist $runname.out.$variable.$step.pfb
 #     }
 # }
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Verify output
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 # source ../../../pftest.tcl
 
@@ -558,21 +560,21 @@
 
 passed = 1
 
-ParflowVariables = [list "satur" "press"]
-step = [format "%05d" 0]
+# ParflowVariables = [list "satur" "press"]
+# step = [format "%05d" 0]
 # foreach variable $ParflowVariables {
-#     set file $runname.out.$variable.$step.pfb 
+#     set file $runname.out.$variable.$step.pfb
 #     if ![pftestFile $file "Max difference in $file" $sig_digits] {
 # 	set passed 0
 #     }
 # }
 
-step = [format "%05d" 12]
+# step = [format "%05d" 12]
 # foreach variable $ClmVariables {
-#     set file $runname.out.$variable.$step.pfb 
-#     if ![pftestFile $file "Max difference in $file" $sig_digits] { 
-# 	set passed 0 
-#     } 
+#     set file $runname.out.$variable.$step.pfb
+#     if ![pftestFile $file "Max difference in $file" $sig_digits] {
+# 	set passed 0
+#     }
 # }
 
 # if $passed {
diff --git a/test/python/water_balance_x.hardflow.jac.py b/test/python/water_balance_x.hardflow.jac.py
index 4131155a5..870a7a6c7 100644
--- a/test/python/water_balance_x.hardflow.jac.py
+++ b/test/python/water_balance_x.hardflow.jac.py
@@ -1,52 +1,63 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  This runs the tilted-v catchment problem
 #  similar to that in Kollet and Maxwell (2006) AWR
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-#---------------------------------------------------------
+
+# ---------------------------------------------------------
+# Import ParFlow
+# ---------------------------------------------------------
+
+import sys
+import os
+import numpy as np
+from parflow import Run
+from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.io import read_pfb, write_pfb, ParflowBinaryReader
+from parflow.tools.top import compute_top
+from parflow.tools import hydrology
+from parflow.tools.compare import pf_test_equal
+
+run_name = "water_balance"
+
+wbx = Run(run_name, __file__)
+
+# ---------------------------------------------------------
 # Some controls for the test
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-#---------------------------------------------------------
-# Control slopes 
-#-1 = slope to lower-x
+# ---------------------------------------------------------
+# Control slopes
+# -1 = slope to lower-x
 # 0 = flat top (no overland flow)
 # 1 = slope to upper-x
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 use_slopes = -1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Flux on the top surface
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rain_flux = -0.05
 rec_flux = 0.0
 
-#---------------------------------------------------------
-# Import ParFlow
-#---------------------------------------------------------
-
-from parflow import Run
-from parflow.tools.fs import mkdir, get_absolute_path
 
-wbx = Run("water_balance_x", __file__)
-
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 wbx.FileVersion = 4
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Processor topology
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 wbx.Process.Topology.P = 1
 wbx.Process.Topology.Q = 1
 wbx.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 wbx.ComputationalGrid.Lower.X = 0.0
 wbx.ComputationalGrid.Lower.Y = 0.0
@@ -60,25 +71,25 @@
 wbx.ComputationalGrid.DY = 10.0
 wbx.ComputationalGrid.DZ = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-wbx.GeomInput.Names = 'domaininput leftinput rightinput channelinput'
+wbx.GeomInput.Names = "domaininput leftinput rightinput channelinput"
 
-wbx.GeomInput.domaininput.GeomName = 'domain'
-wbx.GeomInput.leftinput.GeomName = 'left'
-wbx.GeomInput.rightinput.GeomName = 'right'
-wbx.GeomInput.channelinput.GeomName = 'channel'
+wbx.GeomInput.domaininput.GeomName = "domain"
+wbx.GeomInput.leftinput.GeomName = "left"
+wbx.GeomInput.rightinput.GeomName = "right"
+wbx.GeomInput.channelinput.GeomName = "channel"
 
-wbx.GeomInput.domaininput.InputType = 'Box'
-wbx.GeomInput.leftinput.InputType = 'Box'
-wbx.GeomInput.rightinput.InputType = 'Box'
-wbx.GeomInput.channelinput.InputType = 'Box'
+wbx.GeomInput.domaininput.InputType = "Box"
+wbx.GeomInput.leftinput.InputType = "Box"
+wbx.GeomInput.rightinput.InputType = "Box"
+wbx.GeomInput.channelinput.InputType = "Box"
 
-#---------------------------------------------------------
-# Domain Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Domain Geometry
+# ---------------------------------------------------------
 
 wbx.Geom.domain.Lower.X = 0.0
 wbx.Geom.domain.Lower.Y = 0.0
@@ -87,11 +98,11 @@
 wbx.Geom.domain.Upper.X = 300.0
 wbx.Geom.domain.Upper.Y = 300.0
 wbx.Geom.domain.Upper.Z = 1.5
-wbx.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+wbx.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#---------------------------------------------------------
-# Left Slope Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Left Slope Geometry
+# ---------------------------------------------------------
 
 wbx.Geom.left.Lower.X = 0.0
 wbx.Geom.left.Lower.Y = 0.0
@@ -101,9 +112,9 @@
 wbx.Geom.left.Upper.Y = 140.0
 wbx.Geom.left.Upper.Z = 1.5
 
-#---------------------------------------------------------
-# Right Slope Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Right Slope Geometry
+# ---------------------------------------------------------
 
 wbx.Geom.right.Lower.X = 0.0
 wbx.Geom.right.Lower.Y = 160.0
@@ -113,9 +124,9 @@
 wbx.Geom.right.Upper.Y = 300.0
 wbx.Geom.right.Upper.Z = 1.5
 
-#---------------------------------------------------------
-# Channel Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Channel Geometry
+# ---------------------------------------------------------
 
 wbx.Geom.channel.Lower.X = 0.0
 wbx.Geom.channel.Lower.Y = 140.0
@@ -125,11 +136,11 @@
 wbx.Geom.channel.Upper.Y = 160.0
 wbx.Geom.channel.Upper.Z = 1.5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Geom.Perm.Names = 'left right channel'
+wbx.Geom.Perm.Names = "left right channel"
 
 # Values in m/hour
 
@@ -137,9 +148,9 @@
 # the following is ignored if the perm.type "Constant" settings are not
 # commented out, below.
 
-wbx.Geom.left.Perm.Type = 'TurnBands'
-wbx.Geom.left.Perm.LambdaX = 50.
-wbx.Geom.left.Perm.LambdaY = 50.
+wbx.Geom.left.Perm.Type = "TurnBands"
+wbx.Geom.left.Perm.LambdaX = 50.0
+wbx.Geom.left.Perm.LambdaY = 50.0
 wbx.Geom.left.Perm.LambdaZ = 0.5
 wbx.Geom.left.Perm.GeomMean = 0.01
 
@@ -149,12 +160,12 @@
 wbx.Geom.left.Perm.KMax = 100.0
 wbx.Geom.left.Perm.DelK = 0.2
 wbx.Geom.left.Perm.Seed = 33333
-wbx.Geom.left.Perm.LogNormal = 'Log'
-wbx.Geom.left.Perm.StratType = 'Bottom'
+wbx.Geom.left.Perm.LogNormal = "Log"
+wbx.Geom.left.Perm.StratType = "Bottom"
 
-wbx.Geom.right.Perm.Type = 'TurnBands'
-wbx.Geom.right.Perm.LambdaX = 50.
-wbx.Geom.right.Perm.LambdaY = 50.
+wbx.Geom.right.Perm.Type = "TurnBands"
+wbx.Geom.right.Perm.LambdaX = 50.0
+wbx.Geom.right.Perm.LambdaY = 50.0
 wbx.Geom.right.Perm.LambdaZ = 0.5
 wbx.Geom.right.Perm.GeomMean = 0.05
 
@@ -164,8 +175,8 @@
 wbx.Geom.right.Perm.KMax = 100.0
 wbx.Geom.right.Perm.DelK = 0.2
 wbx.Geom.right.Perm.Seed = 13333
-wbx.Geom.right.Perm.LogNormal = 'Log'
-wbx.Geom.right.Perm.StratType = 'Bottom'
+wbx.Geom.right.Perm.LogNormal = "Log"
+wbx.Geom.right.Perm.StratType = "Bottom"
 
 # hydraulic conductivity is very low, but not zero, top node will have to saturate
 # before overland flow can begin and will be driven by hortonian flow
@@ -173,135 +184,135 @@
 # turning bands above.  Run time increases quite a bit with a heterogeneous
 # subsurface
 
-wbx.Geom.left.Perm.Type = 'Constant'
+wbx.Geom.left.Perm.Type = "Constant"
 wbx.Geom.left.Perm.Value = 0.001
 
-wbx.Geom.right.Perm.Type = 'Constant'
+wbx.Geom.right.Perm.Type = "Constant"
 wbx.Geom.right.Perm.Value = 0.01
 
-wbx.Geom.channel.Perm.Type = 'Constant'
+wbx.Geom.channel.Perm.Type = "Constant"
 wbx.Geom.channel.Perm.Value = 0.00001
 
-wbx.Perm.TensorType = 'TensorByGeom'
+wbx.Perm.TensorType = "TensorByGeom"
 
-wbx.Geom.Perm.TensorByGeom.Names = 'domain'
+wbx.Geom.Perm.TensorByGeom.Names = "domain"
 
 wbx.Geom.domain.Perm.TensorValX = 1.0
 wbx.Geom.domain.Perm.TensorValY = 1.0
 wbx.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.SpecificStorage.Type = 'Constant'
-wbx.SpecificStorage.GeomNames = 'domain'
+wbx.SpecificStorage.Type = "Constant"
+wbx.SpecificStorage.GeomNames = "domain"
 wbx.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Phase.Names = 'water'
+wbx.Phase.Names = "water"
 
-wbx.Phase.water.Density.Type = 'Constant'
+wbx.Phase.water.Density.Type = "Constant"
 wbx.Phase.water.Density.Value = 1.0
 
-wbx.Phase.water.Viscosity.Type = 'Constant'
+wbx.Phase.water.Viscosity.Type = "Constant"
 wbx.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Contaminants.Names = ''
+wbx.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Geom.Retardation.GeomNames = ''
+wbx.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 wbx.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 wbx.TimingInfo.BaseUnit = 0.1
 wbx.TimingInfo.StartCount = 0
 wbx.TimingInfo.StartTime = 0.0
 wbx.TimingInfo.StopTime = 2.0
 wbx.TimingInfo.DumpInterval = 0.1
-wbx.TimeStep.Type = 'Constant'
+wbx.TimeStep.Type = "Constant"
 wbx.TimeStep.Value = 0.1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Geom.Porosity.GeomNames = 'left right channel'
+wbx.Geom.Porosity.GeomNames = "left right channel"
 
-wbx.Geom.left.Porosity.Type = 'Constant'
+wbx.Geom.left.Porosity.Type = "Constant"
 wbx.Geom.left.Porosity.Value = 0.25
 
-wbx.Geom.right.Porosity.Type = 'Constant'
+wbx.Geom.right.Porosity.Type = "Constant"
 wbx.Geom.right.Porosity.Value = 0.25
 
-wbx.Geom.channel.Porosity.Type = 'Constant'
+wbx.Geom.channel.Porosity.Type = "Constant"
 wbx.Geom.channel.Porosity.Value = 0.01
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Domain.GeomName = 'domain'
+wbx.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Phase.RelPerm.Type = 'VanGenuchten'
-wbx.Phase.RelPerm.GeomNames = 'domain'
+wbx.Phase.RelPerm.Type = "VanGenuchten"
+wbx.Phase.RelPerm.GeomNames = "domain"
 
 wbx.Geom.domain.RelPerm.Alpha = 0.5
-wbx.Geom.domain.RelPerm.N = 3.
+wbx.Geom.domain.RelPerm.N = 3.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-wbx.Phase.Saturation.Type = 'VanGenuchten'
-wbx.Phase.Saturation.GeomNames = 'domain'
+wbx.Phase.Saturation.Type = "VanGenuchten"
+wbx.Phase.Saturation.GeomNames = "domain"
 
 wbx.Geom.domain.Saturation.Alpha = 0.5
-wbx.Geom.domain.Saturation.N = 3.
+wbx.Geom.domain.Saturation.N = 3.0
 wbx.Geom.domain.Saturation.SRes = 0.2
 wbx.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Wells.Names = ''
+wbx.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Cycle.Names = 'constant rainrec'
-wbx.Cycle.constant.Names = 'alltime'
+wbx.Cycle.Names = "constant rainrec"
+wbx.Cycle.constant.Names = "alltime"
 wbx.Cycle.constant.alltime.Length = 1
 wbx.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-wbx.Cycle.rainrec.Names = 'r0 r1 r2 r3 r4 r5 r6'
+wbx.Cycle.rainrec.Names = "r0 r1 r2 r3 r4 r5 r6"
 wbx.Cycle.rainrec.r0.Length = 1
 wbx.Cycle.rainrec.r1.Length = 1
 wbx.Cycle.rainrec.r2.Length = 1
@@ -312,33 +323,33 @@
 
 wbx.Cycle.rainrec.Repeat = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 wbx.BCPressure.PatchNames = wbx.Geom.domain.Patches
 
-wbx.Patch.x_lower.BCPressure.Type = 'FluxConst'
-wbx.Patch.x_lower.BCPressure.Cycle = 'constant'
+wbx.Patch.x_lower.BCPressure.Type = "FluxConst"
+wbx.Patch.x_lower.BCPressure.Cycle = "constant"
 wbx.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-wbx.Patch.y_lower.BCPressure.Type = 'FluxConst'
-wbx.Patch.y_lower.BCPressure.Cycle = 'constant'
+wbx.Patch.y_lower.BCPressure.Type = "FluxConst"
+wbx.Patch.y_lower.BCPressure.Cycle = "constant"
 wbx.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-wbx.Patch.z_lower.BCPressure.Type = 'FluxConst'
-wbx.Patch.z_lower.BCPressure.Cycle = 'constant'
+wbx.Patch.z_lower.BCPressure.Type = "FluxConst"
+wbx.Patch.z_lower.BCPressure.Cycle = "constant"
 wbx.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-wbx.Patch.x_upper.BCPressure.Type = 'FluxConst'
-wbx.Patch.x_upper.BCPressure.Cycle = 'constant'
+wbx.Patch.x_upper.BCPressure.Type = "FluxConst"
+wbx.Patch.x_upper.BCPressure.Cycle = "constant"
 wbx.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-wbx.Patch.y_upper.BCPressure.Type = 'FluxConst'
-wbx.Patch.y_upper.BCPressure.Cycle = 'constant'
+wbx.Patch.y_upper.BCPressure.Type = "FluxConst"
+wbx.Patch.y_upper.BCPressure.Cycle = "constant"
 wbx.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-wbx.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-wbx.Patch.z_upper.BCPressure.Cycle = 'rainrec'
+wbx.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+wbx.Patch.z_upper.BCPressure.Cycle = "rainrec"
 wbx.Patch.z_upper.BCPressure.r0.Value = rec_flux
 wbx.Patch.z_upper.BCPressure.r1.Value = rec_flux
 wbx.Patch.z_upper.BCPressure.r2.Value = rain_flux
@@ -347,115 +358,282 @@
 wbx.Patch.z_upper.BCPressure.r5.Value = rec_flux
 wbx.Patch.z_upper.BCPressure.r6.Value = rec_flux
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-wbx.TopoSlopesX.Type = 'Constant'
-wbx.TopoSlopesX.GeomNames = 'left right channel'
+wbx.TopoSlopesX.Type = "Constant"
+wbx.TopoSlopesX.GeomNames = "left right channel"
 if use_slopes > 0:
-  wbx.TopoSlopesX.Geom.left.Value = -0.005
-  wbx.TopoSlopesX.Geom.right.Value = 0.005
-  wbx.TopoSlopesX.Geom.channel.Value = 0.00
+    wbx.TopoSlopesX.Geom.left.Value = -0.005
+    wbx.TopoSlopesX.Geom.right.Value = 0.005
+    wbx.TopoSlopesX.Geom.channel.Value = 0.00
 else:
-  wbx.TopoSlopesX.Geom.left.Value = 0.000
-  wbx.TopoSlopesX.Geom.right.Value = 0.000
-  wbx.TopoSlopesX.Geom.channel.Value = 0.000
+    wbx.TopoSlopesX.Geom.left.Value = 0.000
+    wbx.TopoSlopesX.Geom.right.Value = 0.000
+    wbx.TopoSlopesX.Geom.channel.Value = 0.000
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-wbx.TopoSlopesY.Type = 'Constant'
-wbx.TopoSlopesY.GeomNames = 'left right channel'
+wbx.TopoSlopesY.Type = "Constant"
+wbx.TopoSlopesY.GeomNames = "left right channel"
 if use_slopes > 0:
-  wbx.TopoSlopesY.Geom.left.Value = 0.000
-  wbx.TopoSlopesY.Geom.right.Value = 0.000
-  wbx.TopoSlopesY.Geom.channel.Value = 0.001*use_slopes
+    wbx.TopoSlopesY.Geom.left.Value = 0.000
+    wbx.TopoSlopesY.Geom.right.Value = 0.000
+    wbx.TopoSlopesY.Geom.channel.Value = 0.001 * use_slopes
 else:
-  wbx.TopoSlopesY.Geom.left.Value = 0.000
-  wbx.TopoSlopesY.Geom.right.Value = 0.000
-  wbx.TopoSlopesY.Geom.channel.Value = 0.000
+    wbx.TopoSlopesY.Geom.left.Value = 0.000
+    wbx.TopoSlopesY.Geom.right.Value = 0.000
+    wbx.TopoSlopesY.Geom.channel.Value = 0.000
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-wbx.Mannings.Type = 'Constant'
-wbx.Mannings.GeomNames = 'left right channel'
-wbx.Mannings.Geom.left.Value = 5.e-6
-wbx.Mannings.Geom.right.Value = 5.e-6
-wbx.Mannings.Geom.channel.Value = 1.e-6
+wbx.Mannings.Type = "Constant"
+wbx.Mannings.GeomNames = "left right channel"
+wbx.Mannings.Geom.left.Value = 5.0e-6
+wbx.Mannings.Geom.right.Value = 5.0e-6
+wbx.Mannings.Geom.channel.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.PhaseSources.water.Type = 'Constant'
-wbx.PhaseSources.water.GeomNames = 'domain'
+wbx.PhaseSources.water.Type = "Constant"
+wbx.PhaseSources.water.GeomNames = "domain"
 wbx.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.KnownSolution = 'NoKnownSolution'
+wbx.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Solver = 'Richards'
+wbx.Solver = "Richards"
 wbx.Solver.MaxIter = 2500
 
-wbx.Solver.AbsTol = 1E-10
+wbx.Solver.AbsTol = 1e-10
 wbx.Solver.Nonlinear.MaxIter = 20
 wbx.Solver.Nonlinear.ResidualTol = 1e-9
-wbx.Solver.Nonlinear.EtaChoice = 'Walker1'
-wbx.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+wbx.Solver.Nonlinear.EtaChoice = "Walker1"
+wbx.Solver.Nonlinear.EtaChoice = "EtaConstant"
 wbx.Solver.Nonlinear.EtaValue = 0.01
 wbx.Solver.Nonlinear.UseJacobian = True
 wbx.Solver.Nonlinear.DerivativeEpsilon = 1e-8
 wbx.Solver.Nonlinear.StepTol = 1e-30
-wbx.Solver.Nonlinear.Globalization = 'LineSearch'
+wbx.Solver.Nonlinear.Globalization = "LineSearch"
 wbx.Solver.Linear.KrylovDimension = 20
 wbx.Solver.Linear.MaxRestart = 2
 
-wbx.Solver.Linear.Preconditioner = 'PFMG'
+wbx.Solver.Linear.Preconditioner = "PFMG"
 wbx.Solver.Linear.Preconditioner.PFMG.MaxIter = 1
-wbx.Solver.Linear.Preconditioner.PFMG.Smoother = 'RBGaussSeidelNonSymmetric'
+wbx.Solver.Linear.Preconditioner.PFMG.Smoother = "RBGaussSeidelNonSymmetric"
 wbx.Solver.Linear.Preconditioner.PFMG.NumPreRelax = 1
 wbx.Solver.Linear.Preconditioner.PFMG.NumPostRelax = 1
 
-wbx.Solver.WriteSiloSubsurfData = True
-wbx.Solver.WriteSiloPressure = True
-wbx.Solver.WriteSiloSaturation = True
-wbx.Solver.WriteSiloConcentration = True
-wbx.Solver.WriteSiloSlopes = True
-wbx.Solver.WriteSiloMask = True
-wbx.Solver.WriteSiloEvapTrans = True
-wbx.Solver.WriteSiloEvapTransSum = True
-wbx.Solver.WriteSiloOverlandSum = True
-wbx.Solver.WriteSiloMannings = True
-wbx.Solver.WriteSiloSpecificStorage = True
-
-#---------------------------------------------------------
+wbx.Solver.PrintSubsurfData = True
+wbx.Solver.PrintConcentration = True
+wbx.Solver.PrintSlopes = True
+wbx.Solver.PrintEvapTrans = True
+wbx.Solver.PrintEvapTransSum = True
+wbx.Solver.PrintOverlandSum = True
+wbx.Solver.PrintMannings = True
+wbx.Solver.PrintSpecificStorage = True
+
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be at the bottom of the domain, the top layer is initially dry
-wbx.ICPressure.Type = 'HydroStaticPatch'
-wbx.ICPressure.GeomNames = 'domain'
+wbx.ICPressure.Type = "HydroStaticPatch"
+wbx.ICPressure.GeomNames = "domain"
 
 wbx.Geom.domain.ICPressure.Value = -3.0
 
-wbx.Geom.domain.ICPressure.RefGeom = 'domain'
-wbx.Geom.domain.ICPressure.RefPatch = 'z_upper'
+wbx.Geom.domain.ICPressure.RefGeom = "domain"
+wbx.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/wbx_hj')
-mkdir(dir_name)
-wbx.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+
+new_output_dir_name = get_absolute_path("test_output/water_balance_x")
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+wbx.run(working_directory=new_output_dir_name)
+
+passed = True
+verbose = True
+
+bc_dict = {
+    0: wbx.Patch.z_upper.BCPressure.r0.Value,
+    1: wbx.Patch.z_upper.BCPressure.r1.Value,
+    2: wbx.Patch.z_upper.BCPressure.r2.Value,
+    3: wbx.Patch.z_upper.BCPressure.r3.Value,
+    4: wbx.Patch.z_upper.BCPressure.r4.Value,
+    5: wbx.Patch.z_upper.BCPressure.r5.Value,
+    6: wbx.Patch.z_upper.BCPressure.r6.Value,
+}
+
+slope_x = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.slope_x.pfb"))
+slope_y = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.slope_y.pfb"))
+mannings = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.mannings.pfb"))
+specific_storage = read_pfb(
+    os.path.join(new_output_dir_name, f"{run_name}.out.specific_storage.pfb")
+)
+porosity = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.porosity.pfb"))
+mask = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.mask.pfb"))
+top = compute_top(mask)
+
+surface_area_of_domain = (
+    wbx.ComputationalGrid.DX
+    * wbx.ComputationalGrid.DY
+    * wbx.ComputationalGrid.NX
+    * wbx.ComputationalGrid.NY
+)
+
+prev_total_water_balance = 0.0
+
+for i in range(20):
+    if verbose:
+        print("======================================================")
+        print(f"Timestep {i}")
+        print("======================================================")
+
+    total_water_in_domain = 0.0
+
+    filename = os.path.join(new_output_dir_name, f"{run_name}.out.press.{i:05d}.pfb")
+    pressure = read_pfb(filename)
+    header = ParflowBinaryReader(filename).read_header()
+    dx, dy, dz = header["dx"], header["dy"], header["dz"]
+    surface_storage = hydrology.calculate_surface_storage(pressure, dx, dy, mask)
+    write_pfb(f"surface_storage.{i}.pfb", surface_storage)
+    total_surface_storage = np.sum(surface_storage)
+    if verbose:
+        print(f"Surface storage\t\t\t\t\t : {total_surface_storage:.16e}")
+
+    total_water_in_domain += total_surface_storage
+
+    filename = f"{run_name}.out.satur.{i:05d}.pfb"
+    saturation = read_pfb(os.path.join(new_output_dir_name, filename))
+    water_table_depth = hydrology.calculate_water_table_depth(saturation, top, dz)
+    write_pfb(f"water_table_depth.{i}.pfb", water_table_depth)
+    nz = header["nz"]
+    subsurface_storage = hydrology.calculate_subsurface_storage(
+        porosity,
+        pressure,
+        saturation,
+        specific_storage,
+        dx,
+        dy,
+        np.array([dz] * nz),
+        mask,
+    )
+    write_pfb(f"subsurface_storage.{i}.pfb", subsurface_storage)
+    total_subsurface_storage = np.sum(subsurface_storage)
+    if verbose:
+        print(f"Subsurface storage\t\t\t\t : {total_subsurface_storage:.16e}")
+
+    total_water_in_domain += total_subsurface_storage
+
+    if verbose:
+        print(f"Total water in domain\t\t\t\t : {total_water_in_domain:.16e}")
+        print("")
+
+    total_surface_runoff = 0.0
+    if i > 0:
+        surface_runoff = hydrology.calculate_overland_flow(
+            pressure,
+            slope_x[0],
+            slope_y[0],
+            mannings[0],
+            dx,
+            dy,
+            flow_method="OverlandFlow",
+            mask=mask,
+        )
+        total_surface_runoff = surface_runoff * wbx.TimingInfo.DumpInterval
+        if verbose:
+            print(f"Surface runoff from pftools\t\t\t : {total_surface_runoff:.16e}")
+
+        filename = f"{run_name}.out.overlandsum.{i:05d}.pfb"
+        surface_runoff2 = read_pfb(os.path.join(new_output_dir_name, filename))
+        total_surface_runoff2 = np.sum(surface_runoff2)
+        if verbose:
+            print(
+                f"Surface runoff from pfsimulator\t\t\t : {total_surface_runoff2:.16e}"
+            )
+
+        if not pf_test_equal(
+            total_surface_runoff, total_surface_runoff2, "Surface runoff comparison"
+        ):
+            passed = False
+
+    if i < 1:
+        bc_index = 0
+    elif 1 <= i < 7:
+        bc_index = i - 1
+    else:
+        bc_index = 6
+
+    bc_flux = bc_dict[bc_index]
+    boundary_flux = bc_flux * surface_area_of_domain * wbx.TimingInfo.DumpInterval
+
+    if verbose:
+        print(f"BC flux\t\t\t\t\t\t : {boundary_flux:.16e}")
+
+    expected_difference = boundary_flux + total_surface_runoff
+    if verbose:
+        print(f"Total Flux\t\t\t\t\t : {expected_difference:.16e}")
+
+    if i > 0:
+        if verbose:
+            print("")
+            print(
+                f"Diff from prev total\t\t\t\t : {total_water_in_domain - prev_total_water_balance:.16e}"
+            )
+
+        if expected_difference != 0.0:
+            percent_diff = (
+                abs(
+                    (prev_total_water_balance - total_water_in_domain)
+                    - expected_difference
+                )
+                / abs(expected_difference)
+                * 100
+            )
+            if verbose:
+                print(
+                    f"Percent diff from expected difference\t\t : {percent_diff:.12e}"
+                )
+
+        expected_water_balance = prev_total_water_balance - expected_difference
+        percent_diff = abs(
+            (total_water_in_domain - expected_water_balance)
+            / expected_water_balance
+            * 100
+        )
+        if verbose:
+            print(f"Percent diff from expected total water sum\t : {percent_diff:.12e}")
+
+        if percent_diff > 0.005:
+            print("Error: Water balance is not correct")
+            passed = False
+
+    prev_total_water_balance = total_water_in_domain
+
+if verbose:
+    print("\n\n")
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
diff --git a/test/python/water_balance_x.hardflow.nojac.py b/test/python/water_balance_x.hardflow.nojac.py
index 5912a37e5..b8e6692f0 100644
--- a/test/python/water_balance_x.hardflow.nojac.py
+++ b/test/python/water_balance_x.hardflow.nojac.py
@@ -1,56 +1,65 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  This runs the tilted-v catchment problem
 #  similar to that in Kollet and Maxwell (2006) AWR
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-#---------------------------------------------------------
+
+# ---------------------------------------------------------
+# Import ParFlow
+# ---------------------------------------------------------
+
+import sys
+import os
+import numpy as np
+from parflow import Run
+from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.io import read_pfb, write_pfb, ParflowBinaryReader
+from parflow.tools.top import compute_top
+from parflow.tools import hydrology
+from parflow.tools.compare import pf_test_equal
+
+# ---------------------------------------------------------
+# Name of the run
+# ---------------------------------------------------------
+run_name = "water_balance"
+
+wbx = Run(run_name, __file__)
+
+# ---------------------------------------------------------
 # Some controls for the test
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-#---------------------------------------------------------
-# Control slopes 
+# ---------------------------------------------------------
+# Control slopes
 # -1 = slope to lower-x
 #  0 = flat top (no overland flow)
 #  1 = slope to upper-x
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 use_slopes = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Flux on the top surface
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rain_flux = -0.05
 rec_flux = 0.0
 
-#---------------------------------------------------------
-# Import ParFlow
-#---------------------------------------------------------
-
-from parflow import Run
-from parflow.tools.fs import mkdir, get_absolute_path
-
-#---------------------------------------------------------
-# Name of the run
-#---------------------------------------------------------
-
-wbx = Run("water_balance_x", __file__)
-
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 wbx.FileVersion = 4
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Processor topology
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 wbx.Process.Topology.P = 1
 wbx.Process.Topology.Q = 1
 wbx.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 wbx.ComputationalGrid.Lower.X = 0.0
 wbx.ComputationalGrid.Lower.Y = 0.0
@@ -64,25 +73,25 @@
 wbx.ComputationalGrid.DY = 10.0
 wbx.ComputationalGrid.DZ = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-wbx.GeomInput.Names = 'domaininput leftinput rightinput channelinput'
+wbx.GeomInput.Names = "domaininput leftinput rightinput channelinput"
 
-wbx.GeomInput.domaininput.GeomName = 'domain'
-wbx.GeomInput.leftinput.GeomName = 'left'
-wbx.GeomInput.rightinput.GeomName = 'right'
-wbx.GeomInput.channelinput.GeomName = 'channel'
+wbx.GeomInput.domaininput.GeomName = "domain"
+wbx.GeomInput.leftinput.GeomName = "left"
+wbx.GeomInput.rightinput.GeomName = "right"
+wbx.GeomInput.channelinput.GeomName = "channel"
 
-wbx.GeomInput.domaininput.InputType = 'Box'
-wbx.GeomInput.leftinput.InputType = 'Box'
-wbx.GeomInput.rightinput.InputType = 'Box'
-wbx.GeomInput.channelinput.InputType = 'Box'
+wbx.GeomInput.domaininput.InputType = "Box"
+wbx.GeomInput.leftinput.InputType = "Box"
+wbx.GeomInput.rightinput.InputType = "Box"
+wbx.GeomInput.channelinput.InputType = "Box"
 
-#---------------------------------------------------------
-# Domain Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Domain Geometry
+# ---------------------------------------------------------
 
 wbx.Geom.domain.Lower.X = 0.0
 wbx.Geom.domain.Lower.Y = 0.0
@@ -91,11 +100,11 @@
 wbx.Geom.domain.Upper.X = 300.0
 wbx.Geom.domain.Upper.Y = 300.0
 wbx.Geom.domain.Upper.Z = 1.5
-wbx.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+wbx.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#---------------------------------------------------------
-# Left Slope Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Left Slope Geometry
+# ---------------------------------------------------------
 
 wbx.Geom.left.Lower.X = 0.0
 wbx.Geom.left.Lower.Y = 0.0
@@ -105,9 +114,9 @@
 wbx.Geom.left.Upper.Y = 140.0
 wbx.Geom.left.Upper.Z = 1.5
 
-#---------------------------------------------------------
-# Right Slope Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Right Slope Geometry
+# ---------------------------------------------------------
 
 wbx.Geom.right.Lower.X = 0.0
 wbx.Geom.right.Lower.Y = 160.0
@@ -117,9 +126,9 @@
 wbx.Geom.right.Upper.Y = 300.0
 wbx.Geom.right.Upper.Z = 1.5
 
-#---------------------------------------------------------
-# Channel Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Channel Geometry
+# ---------------------------------------------------------
 
 wbx.Geom.channel.Lower.X = 0.0
 wbx.Geom.channel.Lower.Y = 140.0
@@ -129,11 +138,11 @@
 wbx.Geom.channel.Upper.Y = 160.0
 wbx.Geom.channel.Upper.Z = 1.5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Geom.Perm.Names = 'left right channel'
+wbx.Geom.Perm.Names = "left right channel"
 
 # Values in m/hour
 
@@ -141,9 +150,9 @@
 # the following is ignored if the perm.type "Constant" settings are not
 # commented out, below.
 
-wbx.Geom.left.Perm.Type = 'TurnBands'
-wbx.Geom.left.Perm.LambdaX = 50.
-wbx.Geom.left.Perm.LambdaY = 50.
+wbx.Geom.left.Perm.Type = "TurnBands"
+wbx.Geom.left.Perm.LambdaX = 50.0
+wbx.Geom.left.Perm.LambdaY = 50.0
 wbx.Geom.left.Perm.LambdaZ = 0.5
 wbx.Geom.left.Perm.GeomMean = 0.01
 
@@ -153,12 +162,12 @@
 wbx.Geom.left.Perm.KMax = 100.0
 wbx.Geom.left.Perm.DelK = 0.2
 wbx.Geom.left.Perm.Seed = 33333
-wbx.Geom.left.Perm.LogNormal = 'Log'
-wbx.Geom.left.Perm.StratType = 'Bottom'
+wbx.Geom.left.Perm.LogNormal = "Log"
+wbx.Geom.left.Perm.StratType = "Bottom"
 
-wbx.Geom.right.Perm.Type = 'TurnBands'
-wbx.Geom.right.Perm.LambdaX = 50.
-wbx.Geom.right.Perm.LambdaY = 50.
+wbx.Geom.right.Perm.Type = "TurnBands"
+wbx.Geom.right.Perm.LambdaX = 50.0
+wbx.Geom.right.Perm.LambdaY = 50.0
 wbx.Geom.right.Perm.LambdaZ = 0.5
 wbx.Geom.right.Perm.GeomMean = 0.05
 
@@ -168,8 +177,8 @@
 wbx.Geom.right.Perm.KMax = 100.0
 wbx.Geom.right.Perm.DelK = 0.2
 wbx.Geom.right.Perm.Seed = 13333
-wbx.Geom.right.Perm.LogNormal = 'Log'
-wbx.Geom.right.Perm.StratType = 'Bottom'
+wbx.Geom.right.Perm.LogNormal = "Log"
+wbx.Geom.right.Perm.StratType = "Bottom"
 
 # hydraulic conductivity is very low, but not zero, top node will have to saturate
 # before overland flow can begin and will be driven by hortonian flow
@@ -177,135 +186,135 @@
 # turning bands above.  Run time increases quite a bit with a heterogeneous
 # subsurface
 
-wbx.Geom.left.Perm.Type = 'Constant'
+wbx.Geom.left.Perm.Type = "Constant"
 wbx.Geom.left.Perm.Value = 0.001
 
-wbx.Geom.right.Perm.Type = 'Constant'
+wbx.Geom.right.Perm.Type = "Constant"
 wbx.Geom.right.Perm.Value = 0.01
 
-wbx.Geom.channel.Perm.Type = 'Constant'
+wbx.Geom.channel.Perm.Type = "Constant"
 wbx.Geom.channel.Perm.Value = 0.00001
 
-wbx.Perm.TensorType = 'TensorByGeom'
+wbx.Perm.TensorType = "TensorByGeom"
 
-wbx.Geom.Perm.TensorByGeom.Names = 'domain'
+wbx.Geom.Perm.TensorByGeom.Names = "domain"
 
 wbx.Geom.domain.Perm.TensorValX = 1.0
 wbx.Geom.domain.Perm.TensorValY = 1.0
 wbx.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.SpecificStorage.Type = 'Constant'
-wbx.SpecificStorage.GeomNames = 'domain'
+wbx.SpecificStorage.Type = "Constant"
+wbx.SpecificStorage.GeomNames = "domain"
 wbx.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Phase.Names = 'water'
+wbx.Phase.Names = "water"
 
-wbx.Phase.water.Density.Type = 'Constant'
+wbx.Phase.water.Density.Type = "Constant"
 wbx.Phase.water.Density.Value = 1.0
 
-wbx.Phase.water.Viscosity.Type = 'Constant'
+wbx.Phase.water.Viscosity.Type = "Constant"
 wbx.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Contaminants.Names = ''
+wbx.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Geom.Retardation.GeomNames = ''
+wbx.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 wbx.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 wbx.TimingInfo.BaseUnit = 0.1
 wbx.TimingInfo.StartCount = 0
 wbx.TimingInfo.StartTime = 0.0
 wbx.TimingInfo.StopTime = 2.0
 wbx.TimingInfo.DumpInterval = 0.1
-wbx.TimeStep.Type = 'Constant'
+wbx.TimeStep.Type = "Constant"
 wbx.TimeStep.Value = 0.1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Geom.Porosity.GeomNames = 'left right channel'
+wbx.Geom.Porosity.GeomNames = "left right channel"
 
-wbx.Geom.left.Porosity.Type = 'Constant'
+wbx.Geom.left.Porosity.Type = "Constant"
 wbx.Geom.left.Porosity.Value = 0.25
 
-wbx.Geom.right.Porosity.Type = 'Constant'
+wbx.Geom.right.Porosity.Type = "Constant"
 wbx.Geom.right.Porosity.Value = 0.25
 
-wbx.Geom.channel.Porosity.Type = 'Constant'
+wbx.Geom.channel.Porosity.Type = "Constant"
 wbx.Geom.channel.Porosity.Value = 0.01
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Domain.GeomName = 'domain'
+wbx.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Phase.RelPerm.Type = 'VanGenuchten'
-wbx.Phase.RelPerm.GeomNames = 'domain'
+wbx.Phase.RelPerm.Type = "VanGenuchten"
+wbx.Phase.RelPerm.GeomNames = "domain"
 
 wbx.Geom.domain.RelPerm.Alpha = 0.5
-wbx.Geom.domain.RelPerm.N = 3.
+wbx.Geom.domain.RelPerm.N = 3.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-wbx.Phase.Saturation.Type = 'VanGenuchten'
-wbx.Phase.Saturation.GeomNames = 'domain'
+wbx.Phase.Saturation.Type = "VanGenuchten"
+wbx.Phase.Saturation.GeomNames = "domain"
 
 wbx.Geom.domain.Saturation.Alpha = 0.5
-wbx.Geom.domain.Saturation.N = 3.
+wbx.Geom.domain.Saturation.N = 3.0
 wbx.Geom.domain.Saturation.SRes = 0.2
 wbx.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Wells.Names = ''
+wbx.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Cycle.Names = 'constant rainrec'
-wbx.Cycle.constant.Names = 'alltime'
+wbx.Cycle.Names = "constant rainrec"
+wbx.Cycle.constant.Names = "alltime"
 wbx.Cycle.constant.alltime.Length = 1
 wbx.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-wbx.Cycle.rainrec.Names = 'r0 r1 r2 r3 r4 r5 r6'
+wbx.Cycle.rainrec.Names = "r0 r1 r2 r3 r4 r5 r6"
 wbx.Cycle.rainrec.r0.Length = 1
 wbx.Cycle.rainrec.r1.Length = 1
 wbx.Cycle.rainrec.r2.Length = 1
@@ -315,35 +324,35 @@
 wbx.Cycle.rainrec.r6.Length = 1
 
 wbx.Cycle.rainrec.Repeat = 1
-#  
-#-----------------------------------------------------------------------------
+#
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 wbx.BCPressure.PatchNames = wbx.Geom.domain.Patches
 
-wbx.Patch.x_lower.BCPressure.Type = 'FluxConst'
-wbx.Patch.x_lower.BCPressure.Cycle = 'constant'
+wbx.Patch.x_lower.BCPressure.Type = "FluxConst"
+wbx.Patch.x_lower.BCPressure.Cycle = "constant"
 wbx.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-wbx.Patch.y_lower.BCPressure.Type = 'FluxConst'
-wbx.Patch.y_lower.BCPressure.Cycle = 'constant'
+wbx.Patch.y_lower.BCPressure.Type = "FluxConst"
+wbx.Patch.y_lower.BCPressure.Cycle = "constant"
 wbx.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-wbx.Patch.z_lower.BCPressure.Type = 'FluxConst'
-wbx.Patch.z_lower.BCPressure.Cycle = 'constant'
+wbx.Patch.z_lower.BCPressure.Type = "FluxConst"
+wbx.Patch.z_lower.BCPressure.Cycle = "constant"
 wbx.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-wbx.Patch.x_upper.BCPressure.Type = 'FluxConst'
-wbx.Patch.x_upper.BCPressure.Cycle = 'constant'
+wbx.Patch.x_upper.BCPressure.Type = "FluxConst"
+wbx.Patch.x_upper.BCPressure.Cycle = "constant"
 wbx.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-wbx.Patch.y_upper.BCPressure.Type = 'FluxConst'
-wbx.Patch.y_upper.BCPressure.Cycle = 'constant'
+wbx.Patch.y_upper.BCPressure.Type = "FluxConst"
+wbx.Patch.y_upper.BCPressure.Cycle = "constant"
 wbx.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-wbx.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-wbx.Patch.z_upper.BCPressure.Cycle = 'rainrec'
+wbx.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+wbx.Patch.z_upper.BCPressure.Cycle = "rainrec"
 wbx.Patch.z_upper.BCPressure.r0.Value = rec_flux
 wbx.Patch.z_upper.BCPressure.r1.Value = rec_flux
 wbx.Patch.z_upper.BCPressure.r2.Value = rain_flux
@@ -352,115 +361,281 @@
 wbx.Patch.z_upper.BCPressure.r5.Value = rec_flux
 wbx.Patch.z_upper.BCPressure.r6.Value = rec_flux
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-wbx.TopoSlopesX.Type = 'Constant'
-wbx.TopoSlopesX.GeomNames = 'left right channel'
+wbx.TopoSlopesX.Type = "Constant"
+wbx.TopoSlopesX.GeomNames = "left right channel"
 if use_slopes > 0:
-  wbx.TopoSlopesX.Geom.left.Value = 0.000
-  wbx.TopoSlopesX.Geom.right.Value = 0.000
-  wbx.TopoSlopesX.Geom.channel.Value = 0.001*use_slopes
+    wbx.TopoSlopesX.Geom.left.Value = 0.000
+    wbx.TopoSlopesX.Geom.right.Value = 0.000
+    wbx.TopoSlopesX.Geom.channel.Value = 0.001 * use_slopes
 else:
-  wbx.TopoSlopesX.Geom.left.Value = 0.000
-  wbx.TopoSlopesX.Geom.right.Value = 0.000
-  wbx.TopoSlopesX.Geom.channel.Value = 0.000
+    wbx.TopoSlopesX.Geom.left.Value = 0.000
+    wbx.TopoSlopesX.Geom.right.Value = 0.000
+    wbx.TopoSlopesX.Geom.channel.Value = 0.000
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-wbx.TopoSlopesY.Type = 'Constant'
-wbx.TopoSlopesY.GeomNames = 'left right channel'
+wbx.TopoSlopesY.Type = "Constant"
+wbx.TopoSlopesY.GeomNames = "left right channel"
 if use_slopes > 0:
-  wbx.TopoSlopesY.Geom.left.Value = -0.005
-  wbx.TopoSlopesY.Geom.right.Value = 0.005
-  wbx.TopoSlopesY.Geom.channel.Value = 0.000
+    wbx.TopoSlopesY.Geom.left.Value = -0.005
+    wbx.TopoSlopesY.Geom.right.Value = 0.005
+    wbx.TopoSlopesY.Geom.channel.Value = 0.000
 else:
-  wbx.TopoSlopesY.Geom.left.Value = 0.000
-  wbx.TopoSlopesY.Geom.right.Value = 0.000
-  wbx.TopoSlopesY.Geom.channel.Value = 0.000
+    wbx.TopoSlopesY.Geom.left.Value = 0.000
+    wbx.TopoSlopesY.Geom.right.Value = 0.000
+    wbx.TopoSlopesY.Geom.channel.Value = 0.000
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-wbx.Mannings.Type = 'Constant'
-wbx.Mannings.GeomNames = 'left right channel'
-wbx.Mannings.Geom.left.Value = 5.e-6
-wbx.Mannings.Geom.right.Value = 5.e-6
-wbx.Mannings.Geom.channel.Value = 1.e-6
+wbx.Mannings.Type = "Constant"
+wbx.Mannings.GeomNames = "left right channel"
+wbx.Mannings.Geom.left.Value = 5.0e-6
+wbx.Mannings.Geom.right.Value = 5.0e-6
+wbx.Mannings.Geom.channel.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.PhaseSources.water.Type = 'Constant'
-wbx.PhaseSources.water.GeomNames = 'domain'
+wbx.PhaseSources.water.Type = "Constant"
+wbx.PhaseSources.water.GeomNames = "domain"
 wbx.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.KnownSolution = 'NoKnownSolution'
+wbx.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Solver = 'Richards'
+wbx.Solver = "Richards"
 wbx.Solver.MaxIter = 2500
 
-wbx.Solver.AbsTol = 1E-10
+wbx.Solver.AbsTol = 1e-10
 wbx.Solver.Nonlinear.MaxIter = 20
 wbx.Solver.Nonlinear.ResidualTol = 1e-9
-wbx.Solver.Nonlinear.EtaChoice = 'Walker1'
-wbx.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+wbx.Solver.Nonlinear.EtaChoice = "Walker1"
+wbx.Solver.Nonlinear.EtaChoice = "EtaConstant"
 wbx.Solver.Nonlinear.EtaValue = 0.01
 wbx.Solver.Nonlinear.UseJacobian = False
 wbx.Solver.Nonlinear.DerivativeEpsilon = 1e-8
 wbx.Solver.Nonlinear.StepTol = 1e-30
-wbx.Solver.Nonlinear.Globalization = 'LineSearch'
+wbx.Solver.Nonlinear.Globalization = "LineSearch"
 wbx.Solver.Linear.KrylovDimension = 20
 wbx.Solver.Linear.MaxRestart = 2
 
-wbx.Solver.Linear.Preconditioner = 'PFMG'
+wbx.Solver.Linear.Preconditioner = "PFMG"
 wbx.Solver.Linear.Preconditioner.PFMG.MaxIter = 1
-wbx.Solver.Linear.Preconditioner.PFMG.Smoother = 'RBGaussSeidelNonSymmetric'
+wbx.Solver.Linear.Preconditioner.PFMG.Smoother = "RBGaussSeidelNonSymmetric"
 wbx.Solver.Linear.Preconditioner.PFMG.NumPreRelax = 1
 wbx.Solver.Linear.Preconditioner.PFMG.NumPostRelax = 1
 
-wbx.Solver.WriteSiloSubsurfData = True
-wbx.Solver.WriteSiloPressure = True
-wbx.Solver.WriteSiloSaturation = True
-wbx.Solver.WriteSiloConcentration = True
-wbx.Solver.WriteSiloSlopes = True
-wbx.Solver.WriteSiloMask = True
-wbx.Solver.WriteSiloEvapTrans = True
-wbx.Solver.WriteSiloEvapTransSum = True
-wbx.Solver.WriteSiloOverlandSum = True
-wbx.Solver.WriteSiloMannings = True
-wbx.Solver.WriteSiloSpecificStorage = True
-
-#---------------------------------------------------------
+wbx.Solver.PrintSubsurfData = True
+wbx.Solver.PrintConcentration = True
+wbx.Solver.PrintSlopes = True
+wbx.Solver.PrintEvapTrans = True
+wbx.Solver.PrintEvapTransSum = True
+wbx.Solver.PrintOverlandSum = True
+wbx.Solver.PrintMannings = True
+wbx.Solver.PrintSpecificStorage = True
+
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be at the bottom of the domain, the top layer is initially dry
-wbx.ICPressure.Type = 'HydroStaticPatch'
-wbx.ICPressure.GeomNames = 'domain'
+wbx.ICPressure.Type = "HydroStaticPatch"
+wbx.ICPressure.GeomNames = "domain"
 
 wbx.Geom.domain.ICPressure.Value = -3.0
 
-wbx.Geom.domain.ICPressure.RefGeom = 'domain'
-wbx.Geom.domain.ICPressure.RefPatch = 'z_upper'
+wbx.Geom.domain.ICPressure.RefGeom = "domain"
+wbx.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/wbx_hnj')
-mkdir(dir_name)
-wbx.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/water_balance_x")
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+wbx.run(working_directory=new_output_dir_name)
+
+passed = True
+verbose = True
+
+bc_dict = {
+    0: wbx.Patch.z_upper.BCPressure.r0.Value,
+    1: wbx.Patch.z_upper.BCPressure.r1.Value,
+    2: wbx.Patch.z_upper.BCPressure.r2.Value,
+    3: wbx.Patch.z_upper.BCPressure.r3.Value,
+    4: wbx.Patch.z_upper.BCPressure.r4.Value,
+    5: wbx.Patch.z_upper.BCPressure.r5.Value,
+    6: wbx.Patch.z_upper.BCPressure.r6.Value,
+}
+
+slope_x = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.slope_x.pfb"))
+slope_y = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.slope_y.pfb"))
+mannings = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.mannings.pfb"))
+specific_storage = read_pfb(
+    os.path.join(new_output_dir_name, f"{run_name}.out.specific_storage.pfb")
+)
+porosity = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.porosity.pfb"))
+mask = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.mask.pfb"))
+top = compute_top(mask)
+
+surface_area_of_domain = (
+    wbx.ComputationalGrid.DX
+    * wbx.ComputationalGrid.DY
+    * wbx.ComputationalGrid.NX
+    * wbx.ComputationalGrid.NY
+)
+
+prev_total_water_balance = 0.0
+
+for i in range(20):
+    if verbose:
+        print("======================================================")
+        print(f"Timestep {i}")
+        print("======================================================")
+
+    total_water_in_domain = 0.0
+
+    filename = os.path.join(new_output_dir_name, f"{run_name}.out.press.{i:05d}.pfb")
+    pressure = read_pfb(filename)
+    header = ParflowBinaryReader(filename).read_header()
+    dx, dy, dz = header["dx"], header["dy"], header["dz"]
+    surface_storage = hydrology.calculate_surface_storage(pressure, dx, dy, mask)
+    write_pfb(f"surface_storage.{i}.pfb", surface_storage)
+    total_surface_storage = np.sum(surface_storage)
+    if verbose:
+        print(f"Surface storage\t\t\t\t\t : {total_surface_storage:.16e}")
+
+    total_water_in_domain += total_surface_storage
+
+    filename = f"{run_name}.out.satur.{i:05d}.pfb"
+    saturation = read_pfb(os.path.join(new_output_dir_name, filename))
+    water_table_depth = hydrology.calculate_water_table_depth(saturation, top, dz)
+    write_pfb(f"water_table_depth.{i}.pfb", water_table_depth)
+    nz = header["nz"]
+    subsurface_storage = hydrology.calculate_subsurface_storage(
+        porosity,
+        pressure,
+        saturation,
+        specific_storage,
+        dx,
+        dy,
+        np.array([dz] * nz),
+        mask,
+    )
+    write_pfb(f"subsurface_storage.{i}.pfb", subsurface_storage)
+    total_subsurface_storage = np.sum(subsurface_storage)
+    if verbose:
+        print(f"Subsurface storage\t\t\t\t : {total_subsurface_storage:.16e}")
+
+    total_water_in_domain += total_subsurface_storage
+
+    if verbose:
+        print(f"Total water in domain\t\t\t\t : {total_water_in_domain:.16e}")
+        print("")
+
+    total_surface_runoff = 0.0
+    if i > 0:
+        surface_runoff = hydrology.calculate_overland_flow(
+            pressure,
+            slope_x[0],
+            slope_y[0],
+            mannings[0],
+            dx,
+            dy,
+            flow_method="OverlandFlow",
+            mask=mask,
+        )
+        total_surface_runoff = surface_runoff * wbx.TimingInfo.DumpInterval
+        if verbose:
+            print(f"Surface runoff from pftools\t\t\t : {total_surface_runoff:.16e}")
+
+        filename = f"{run_name}.out.overlandsum.{i:05d}.pfb"
+        surface_runoff2 = read_pfb(os.path.join(new_output_dir_name, filename))
+        total_surface_runoff2 = np.sum(surface_runoff2)
+        if verbose:
+            print(
+                f"Surface runoff from pfsimulator\t\t\t : {total_surface_runoff2:.16e}"
+            )
+
+        if not pf_test_equal(
+            total_surface_runoff, total_surface_runoff2, "Surface runoff comparison"
+        ):
+            passed = False
+
+    if i < 1:
+        bc_index = 0
+    elif 1 <= i < 7:
+        bc_index = i - 1
+    else:
+        bc_index = 6
+
+    bc_flux = bc_dict[bc_index]
+    boundary_flux = bc_flux * surface_area_of_domain * wbx.TimingInfo.DumpInterval
+
+    if verbose:
+        print(f"BC flux\t\t\t\t\t\t : {boundary_flux:.16e}")
+
+    expected_difference = boundary_flux + total_surface_runoff
+    if verbose:
+        print(f"Total Flux\t\t\t\t\t : {expected_difference:.16e}")
+
+    if i > 0:
+        if verbose:
+            print("")
+            print(
+                f"Diff from prev total\t\t\t\t : {total_water_in_domain - prev_total_water_balance:.16e}"
+            )
+
+        if expected_difference != 0.0:
+            percent_diff = (
+                abs(
+                    (prev_total_water_balance - total_water_in_domain)
+                    - expected_difference
+                )
+                / abs(expected_difference)
+                * 100
+            )
+            if verbose:
+                print(
+                    f"Percent diff from expected difference\t\t : {percent_diff:.12e}"
+                )
+
+        expected_water_balance = prev_total_water_balance - expected_difference
+        percent_diff = abs(
+            (total_water_in_domain - expected_water_balance)
+            / expected_water_balance
+            * 100
+        )
+        if verbose:
+            print(f"Percent diff from expected total water sum\t : {percent_diff:.12e}")
+
+        if percent_diff > 0.005:
+            print("Error: Water balance is not correct")
+            passed = False
+
+    prev_total_water_balance = total_water_in_domain
+
+if verbose:
+    print("\n\n")
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
diff --git a/test/python/water_balance_x.py b/test/python/water_balance_x.py
index 9174247fb..a2f2f1911 100644
--- a/test/python/water_balance_x.py
+++ b/test/python/water_balance_x.py
@@ -1,52 +1,62 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  This runs the tilted-v catchment problem
 #  similar to that in Kollet and Maxwell (2006) AWR
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Import ParFlow
+# ---------------------------------------------------------
+
+import sys
+import os
+import numpy as np
+from parflow import Run
+from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.io import read_pfb, write_pfb, ParflowBinaryReader
+from parflow.tools.top import compute_top
+from parflow.tools import hydrology
+from parflow.tools.compare import pf_test_equal
+
+run_name = "water_balance"
+
+wbx = Run(run_name, __file__)
+
+
+# ---------------------------------------------------------
 # Some controls for the test
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-#---------------------------------------------------------
-# Control slopes 
+# ---------------------------------------------------------
+# Control slopes
 # -1 = slope to lower-x
 #  0 = flat top (no overland flow)
 #  1 = slope to upper-x
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 use_slopes = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Flux on the top surface
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rain_flux = -0.05
 rec_flux = 0.0
 
-#---------------------------------------------------------
-# Import ParFlow
-#---------------------------------------------------------
-
-from parflow import Run
-from parflow.tools.fs import mkdir, get_absolute_path
-
-wbx = Run("wbx", __file__)
-
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 wbx.FileVersion = 4
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Processor topology
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 wbx.Process.Topology.P = 1
 wbx.Process.Topology.Q = 1
 wbx.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 wbx.ComputationalGrid.Lower.X = 0.0
 wbx.ComputationalGrid.Lower.Y = 0.0
@@ -60,25 +70,25 @@
 wbx.ComputationalGrid.DY = 10.0
 wbx.ComputationalGrid.DZ = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-wbx.GeomInput.Names = 'domaininput leftinput rightinput channelinput'
+wbx.GeomInput.Names = "domaininput leftinput rightinput channelinput"
 
-wbx.GeomInput.domaininput.GeomName = 'domain'
-wbx.GeomInput.leftinput.GeomName = 'left'
-wbx.GeomInput.rightinput.GeomName = 'right'
-wbx.GeomInput.channelinput.GeomName = 'channel'
+wbx.GeomInput.domaininput.GeomName = "domain"
+wbx.GeomInput.leftinput.GeomName = "left"
+wbx.GeomInput.rightinput.GeomName = "right"
+wbx.GeomInput.channelinput.GeomName = "channel"
 
-wbx.GeomInput.domaininput.InputType = 'Box'
-wbx.GeomInput.leftinput.InputType = 'Box'
-wbx.GeomInput.rightinput.InputType = 'Box'
-wbx.GeomInput.channelinput.InputType = 'Box'
+wbx.GeomInput.domaininput.InputType = "Box"
+wbx.GeomInput.leftinput.InputType = "Box"
+wbx.GeomInput.rightinput.InputType = "Box"
+wbx.GeomInput.channelinput.InputType = "Box"
 
-#---------------------------------------------------------
-# Domain Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Domain Geometry
+# ---------------------------------------------------------
 
 wbx.Geom.domain.Lower.X = 0.0
 wbx.Geom.domain.Lower.Y = 0.0
@@ -86,11 +96,11 @@
 wbx.Geom.domain.Upper.X = 300.0
 wbx.Geom.domain.Upper.Y = 300.0
 wbx.Geom.domain.Upper.Z = 1.5
-wbx.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+wbx.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#---------------------------------------------------------
-# Left Slope Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Left Slope Geometry
+# ---------------------------------------------------------
 
 wbx.Geom.left.Lower.X = 0.0
 wbx.Geom.left.Lower.Y = 0.0
@@ -99,9 +109,9 @@
 wbx.Geom.left.Upper.Y = 140.0
 wbx.Geom.left.Upper.Z = 1.5
 
-#---------------------------------------------------------
-# Right Slope Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Right Slope Geometry
+# ---------------------------------------------------------
 
 wbx.Geom.right.Lower.X = 0.0
 wbx.Geom.right.Lower.Y = 160.0
@@ -110,9 +120,9 @@
 wbx.Geom.right.Upper.Y = 300.0
 wbx.Geom.right.Upper.Z = 1.5
 
-#---------------------------------------------------------
-# Channel Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Channel Geometry
+# ---------------------------------------------------------
 
 wbx.Geom.channel.Lower.X = 0.0
 wbx.Geom.channel.Lower.Y = 140.0
@@ -121,11 +131,11 @@
 wbx.Geom.channel.Upper.Y = 160.0
 wbx.Geom.channel.Upper.Z = 1.5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Geom.Perm.Names = 'left right channel'
+wbx.Geom.Perm.Names = "left right channel"
 
 # Values in m/hour
 
@@ -133,9 +143,9 @@
 # the following is ignored if the perm.type "Constant" settings are not
 # commented out, below.
 
-wbx.Geom.left.Perm.Type = 'TurnBands'
-wbx.Geom.left.Perm.LambdaX = 50.
-wbx.Geom.left.Perm.LambdaY = 50.
+wbx.Geom.left.Perm.Type = "TurnBands"
+wbx.Geom.left.Perm.LambdaX = 50.0
+wbx.Geom.left.Perm.LambdaY = 50.0
 wbx.Geom.left.Perm.LambdaZ = 0.5
 wbx.Geom.left.Perm.GeomMean = 0.01
 
@@ -145,12 +155,12 @@
 wbx.Geom.left.Perm.KMax = 100.0
 wbx.Geom.left.Perm.DelK = 0.2
 wbx.Geom.left.Perm.Seed = 33333
-wbx.Geom.left.Perm.LogNormal = 'Log'
-wbx.Geom.left.Perm.StratType = 'Bottom'
+wbx.Geom.left.Perm.LogNormal = "Log"
+wbx.Geom.left.Perm.StratType = "Bottom"
 
-wbx.Geom.right.Perm.Type = 'TurnBands'
-wbx.Geom.right.Perm.LambdaX = 50.
-wbx.Geom.right.Perm.LambdaY = 50.
+wbx.Geom.right.Perm.Type = "TurnBands"
+wbx.Geom.right.Perm.LambdaX = 50.0
+wbx.Geom.right.Perm.LambdaY = 50.0
 wbx.Geom.right.Perm.LambdaZ = 0.5
 wbx.Geom.right.Perm.GeomMean = 0.05
 
@@ -160,8 +170,8 @@
 wbx.Geom.right.Perm.KMax = 100.0
 wbx.Geom.right.Perm.DelK = 0.2
 wbx.Geom.right.Perm.Seed = 13333
-wbx.Geom.right.Perm.LogNormal = 'Log'
-wbx.Geom.right.Perm.StratType = 'Bottom'
+wbx.Geom.right.Perm.LogNormal = "Log"
+wbx.Geom.right.Perm.StratType = "Bottom"
 
 # hydraulic conductivity is very low, but not zero, top node will have to saturate
 # before overland flow can begin and will be driven by hortonian flow
@@ -169,135 +179,135 @@
 # turning bands above.  Run time increases quite a bit with a heterogeneous
 # subsurface
 
-wbx.Geom.left.Perm.Type = 'Constant'
+wbx.Geom.left.Perm.Type = "Constant"
 wbx.Geom.left.Perm.Value = 0.001
 
-wbx.Geom.right.Perm.Type = 'Constant'
+wbx.Geom.right.Perm.Type = "Constant"
 wbx.Geom.right.Perm.Value = 0.01
 
-wbx.Geom.channel.Perm.Type = 'Constant'
+wbx.Geom.channel.Perm.Type = "Constant"
 wbx.Geom.channel.Perm.Value = 0.00001
 
-wbx.Perm.TensorType = 'TensorByGeom'
+wbx.Perm.TensorType = "TensorByGeom"
 
-wbx.Geom.Perm.TensorByGeom.Names = 'domain'
+wbx.Geom.Perm.TensorByGeom.Names = "domain"
 
 wbx.Geom.domain.Perm.TensorValX = 1.0
 wbx.Geom.domain.Perm.TensorValY = 1.0
 wbx.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.SpecificStorage.Type = 'Constant'
-wbx.SpecificStorage.GeomNames = 'domain'
+wbx.SpecificStorage.Type = "Constant"
+wbx.SpecificStorage.GeomNames = "domain"
 wbx.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Phase.Names = 'water'
+wbx.Phase.Names = "water"
 
-wbx.Phase.water.Density.Type = 'Constant'
+wbx.Phase.water.Density.Type = "Constant"
 wbx.Phase.water.Density.Value = 1.0
 
-wbx.Phase.water.Viscosity.Type = 'Constant'
+wbx.Phase.water.Viscosity.Type = "Constant"
 wbx.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Contaminants.Names = ''
+wbx.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Geom.Retardation.GeomNames = ''
+wbx.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 wbx.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 wbx.TimingInfo.BaseUnit = 0.1
 wbx.TimingInfo.StartCount = 0
 wbx.TimingInfo.StartTime = 0.0
 wbx.TimingInfo.StopTime = 2.0
 wbx.TimingInfo.DumpInterval = 0.1
-wbx.TimeStep.Type = 'Constant'
+wbx.TimeStep.Type = "Constant"
 wbx.TimeStep.Value = 0.1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Geom.Porosity.GeomNames = 'left right channel'
+wbx.Geom.Porosity.GeomNames = "left right channel"
 
-wbx.Geom.left.Porosity.Type = 'Constant'
+wbx.Geom.left.Porosity.Type = "Constant"
 wbx.Geom.left.Porosity.Value = 0.25
 
-wbx.Geom.right.Porosity.Type = 'Constant'
+wbx.Geom.right.Porosity.Type = "Constant"
 wbx.Geom.right.Porosity.Value = 0.25
 
-wbx.Geom.channel.Porosity.Type = 'Constant'
+wbx.Geom.channel.Porosity.Type = "Constant"
 wbx.Geom.channel.Porosity.Value = 0.01
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Domain.GeomName = 'domain'
+wbx.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Phase.RelPerm.Type = 'VanGenuchten'
-wbx.Phase.RelPerm.GeomNames = 'domain'
+wbx.Phase.RelPerm.Type = "VanGenuchten"
+wbx.Phase.RelPerm.GeomNames = "domain"
 
 wbx.Geom.domain.RelPerm.Alpha = 0.5
-wbx.Geom.domain.RelPerm.N = 3.
+wbx.Geom.domain.RelPerm.N = 3.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-wbx.Phase.Saturation.Type = 'VanGenuchten'
-wbx.Phase.Saturation.GeomNames = 'domain'
+wbx.Phase.Saturation.Type = "VanGenuchten"
+wbx.Phase.Saturation.GeomNames = "domain"
 
 wbx.Geom.domain.Saturation.Alpha = 0.5
-wbx.Geom.domain.Saturation.N = 3.
+wbx.Geom.domain.Saturation.N = 3.0
 wbx.Geom.domain.Saturation.SRes = 0.2
 wbx.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Wells.Names = ''
+wbx.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Cycle.Names = 'constant rainrec'
-wbx.Cycle.constant.Names = 'alltime'
+wbx.Cycle.Names = "constant rainrec"
+wbx.Cycle.constant.Names = "alltime"
 wbx.Cycle.constant.alltime.Length = 1
 wbx.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-wbx.Cycle.rainrec.Names = 'r0 r1 r2 r3 r4 r5 r6'
+wbx.Cycle.rainrec.Names = "r0 r1 r2 r3 r4 r5 r6"
 wbx.Cycle.rainrec.r0.Length = 1
 wbx.Cycle.rainrec.r1.Length = 1
 wbx.Cycle.rainrec.r2.Length = 1
@@ -308,35 +318,35 @@
 
 wbx.Cycle.rainrec.Repeat = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 wbx.BCPressure.PatchNames = wbx.Geom.domain.Patches
 
-wbx.Patch.x_lower.BCPressure.Type = 'FluxConst'
-wbx.Patch.x_lower.BCPressure.Cycle = 'constant'
+wbx.Patch.x_lower.BCPressure.Type = "FluxConst"
+wbx.Patch.x_lower.BCPressure.Cycle = "constant"
 wbx.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-wbx.Patch.y_lower.BCPressure.Type = 'FluxConst'
-wbx.Patch.y_lower.BCPressure.Cycle = 'constant'
+wbx.Patch.y_lower.BCPressure.Type = "FluxConst"
+wbx.Patch.y_lower.BCPressure.Cycle = "constant"
 wbx.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-wbx.Patch.z_lower.BCPressure.Type = 'FluxConst'
-wbx.Patch.z_lower.BCPressure.Cycle = 'constant'
+wbx.Patch.z_lower.BCPressure.Type = "FluxConst"
+wbx.Patch.z_lower.BCPressure.Cycle = "constant"
 wbx.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-wbx.Patch.x_upper.BCPressure.Type = 'FluxConst'
-wbx.Patch.x_upper.BCPressure.Cycle = 'constant'
+wbx.Patch.x_upper.BCPressure.Type = "FluxConst"
+wbx.Patch.x_upper.BCPressure.Cycle = "constant"
 wbx.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-wbx.Patch.y_upper.BCPressure.Type = 'FluxConst'
-wbx.Patch.y_upper.BCPressure.Cycle = 'constant'
+wbx.Patch.y_upper.BCPressure.Type = "FluxConst"
+wbx.Patch.y_upper.BCPressure.Cycle = "constant"
 wbx.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
 
-wbx.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-wbx.Patch.z_upper.BCPressure.Cycle = 'rainrec'
+wbx.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+wbx.Patch.z_upper.BCPressure.Cycle = "rainrec"
 wbx.Patch.z_upper.BCPressure.r0.Value = rec_flux
 wbx.Patch.z_upper.BCPressure.r1.Value = rec_flux
 wbx.Patch.z_upper.BCPressure.r2.Value = rain_flux
@@ -345,115 +355,281 @@
 wbx.Patch.z_upper.BCPressure.r5.Value = rec_flux
 wbx.Patch.z_upper.BCPressure.r6.Value = rec_flux
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-wbx.TopoSlopesX.Type = 'Constant'
-wbx.TopoSlopesX.GeomNames = 'left right channel'
+wbx.TopoSlopesX.Type = "Constant"
+wbx.TopoSlopesX.GeomNames = "left right channel"
 if use_slopes > 0:
-  wbx.TopoSlopesX.Geom.left.Value = 0.000
-  wbx.TopoSlopesX.Geom.right.Value = 0.000
-  wbx.TopoSlopesX.Geom.channel.Value = 0.001*use_slopes
+    wbx.TopoSlopesX.Geom.left.Value = 0.000
+    wbx.TopoSlopesX.Geom.right.Value = 0.000
+    wbx.TopoSlopesX.Geom.channel.Value = 0.001 * use_slopes
 else:
-  wbx.TopoSlopesX.Geom.left.Value = 0.000
-  wbx.TopoSlopesX.Geom.right.Value = 0.000
-  wbx.TopoSlopesX.Geom.channel.Value = 0.000
+    wbx.TopoSlopesX.Geom.left.Value = 0.000
+    wbx.TopoSlopesX.Geom.right.Value = 0.000
+    wbx.TopoSlopesX.Geom.channel.Value = 0.000
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-wbx.TopoSlopesY.Type = 'Constant'
-wbx.TopoSlopesY.GeomNames = 'left right channel'
+wbx.TopoSlopesY.Type = "Constant"
+wbx.TopoSlopesY.GeomNames = "left right channel"
 if use_slopes > 0:
-  wbx.TopoSlopesY.Geom.left.Value = -0.005
-  wbx.TopoSlopesY.Geom.right.Value = 0.005
-  wbx.TopoSlopesY.Geom.channel.Value = 0.000
+    wbx.TopoSlopesY.Geom.left.Value = -0.005
+    wbx.TopoSlopesY.Geom.right.Value = 0.005
+    wbx.TopoSlopesY.Geom.channel.Value = 0.000
 else:
-  wbx.TopoSlopesY.Geom.left.Value = 0.000
-  wbx.TopoSlopesY.Geom.right.Value = 0.000
-  wbx.TopoSlopesY.Geom.channel.Value = 0.000
+    wbx.TopoSlopesY.Geom.left.Value = 0.000
+    wbx.TopoSlopesY.Geom.right.Value = 0.000
+    wbx.TopoSlopesY.Geom.channel.Value = 0.000
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-wbx.Mannings.Type = 'Constant'
-wbx.Mannings.GeomNames = 'left right channel'
-wbx.Mannings.Geom.left.Value = 5.e-6
-wbx.Mannings.Geom.right.Value = 5.e-6
-wbx.Mannings.Geom.channel.Value = 1.e-6
+wbx.Mannings.Type = "Constant"
+wbx.Mannings.GeomNames = "left right channel"
+wbx.Mannings.Geom.left.Value = 5.0e-6
+wbx.Mannings.Geom.right.Value = 5.0e-6
+wbx.Mannings.Geom.channel.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.PhaseSources.water.Type = 'Constant'
-wbx.PhaseSources.water.GeomNames = 'domain'
+wbx.PhaseSources.water.Type = "Constant"
+wbx.PhaseSources.water.GeomNames = "domain"
 wbx.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.KnownSolution = 'NoKnownSolution'
+wbx.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wbx.Solver = 'Richards'
+wbx.Solver = "Richards"
 wbx.Solver.MaxIter = 100
 
-wbx.Solver.AbsTol = 1E-10
+wbx.Solver.AbsTol = 1e-10
 wbx.Solver.Nonlinear.MaxIter = 20
 wbx.Solver.Nonlinear.ResidualTol = 1e-9
-wbx.Solver.Nonlinear.EtaChoice = 'Walker1'
-wbx.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+wbx.Solver.Nonlinear.EtaChoice = "Walker1"
+wbx.Solver.Nonlinear.EtaChoice = "EtaConstant"
 wbx.Solver.Nonlinear.EtaValue = 0.01
 wbx.Solver.Nonlinear.UseJacobian = False
 wbx.Solver.Nonlinear.DerivativeEpsilon = 1e-8
 wbx.Solver.Nonlinear.StepTol = 1e-30
-wbx.Solver.Nonlinear.Globalization = 'LineSearch'
+wbx.Solver.Nonlinear.Globalization = "LineSearch"
 wbx.Solver.Linear.KrylovDimension = 20
 wbx.Solver.Linear.MaxRestart = 2
 
-wbx.Solver.Linear.Preconditioner = 'PFMG'
+wbx.Solver.Linear.Preconditioner = "PFMG"
 wbx.Solver.Linear.Preconditioner.PFMG.MaxIter = 1
-wbx.Solver.Linear.Preconditioner.PFMG.Smoother = 'RBGaussSeidelNonSymmetric'
+wbx.Solver.Linear.Preconditioner.PFMG.Smoother = "RBGaussSeidelNonSymmetric"
 wbx.Solver.Linear.Preconditioner.PFMG.NumPreRelax = 1
 wbx.Solver.Linear.Preconditioner.PFMG.NumPostRelax = 1
 
-wbx.Solver.WriteSiloSubsurfData = True
-wbx.Solver.WriteSiloPressure = True
-wbx.Solver.WriteSiloSaturation = True
-wbx.Solver.WriteSiloConcentration = True
-wbx.Solver.WriteSiloSlopes = True
-wbx.Solver.WriteSiloMask = True
-wbx.Solver.WriteSiloEvapTrans = True
-wbx.Solver.WriteSiloEvapTransSum = True
-wbx.Solver.WriteSiloOverlandSum = True
-wbx.Solver.WriteSiloMannings = True
-wbx.Solver.WriteSiloSpecificStorage = True
-
-#---------------------------------------------------------
+wbx.Solver.PrintSubsurfData = True
+wbx.Solver.PrintConcentration = True
+wbx.Solver.PrintSlopes = True
+wbx.Solver.PrintEvapTrans = True
+wbx.Solver.PrintEvapTransSum = True
+wbx.Solver.PrintOverlandSum = True
+wbx.Solver.PrintMannings = True
+wbx.Solver.PrintSpecificStorage = True
+
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be at the bottom of the domain, the top layer is initially dry
-wbx.ICPressure.Type = 'HydroStaticPatch'
-wbx.ICPressure.GeomNames = 'domain'
+wbx.ICPressure.Type = "HydroStaticPatch"
+wbx.ICPressure.GeomNames = "domain"
 
 wbx.Geom.domain.ICPressure.Value = -3.0
 
-wbx.Geom.domain.ICPressure.RefGeom = 'domain'
-wbx.Geom.domain.ICPressure.RefPatch = 'z_upper'
+wbx.Geom.domain.ICPressure.RefGeom = "domain"
+wbx.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/wbx')
-mkdir(dir_name)
-wbx.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/water_balance_x")
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+wbx.run(working_directory=new_output_dir_name)
+
+passed = True
+verbose = True
+
+bc_dict = {
+    0: wbx.Patch.z_upper.BCPressure.r0.Value,
+    1: wbx.Patch.z_upper.BCPressure.r1.Value,
+    2: wbx.Patch.z_upper.BCPressure.r2.Value,
+    3: wbx.Patch.z_upper.BCPressure.r3.Value,
+    4: wbx.Patch.z_upper.BCPressure.r4.Value,
+    5: wbx.Patch.z_upper.BCPressure.r5.Value,
+    6: wbx.Patch.z_upper.BCPressure.r6.Value,
+}
+
+slope_x = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.slope_x.pfb"))
+slope_y = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.slope_y.pfb"))
+mannings = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.mannings.pfb"))
+specific_storage = read_pfb(
+    os.path.join(new_output_dir_name, f"{run_name}.out.specific_storage.pfb")
+)
+porosity = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.porosity.pfb"))
+mask = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.mask.pfb"))
+top = compute_top(mask)
+
+surface_area_of_domain = (
+    wbx.ComputationalGrid.DX
+    * wbx.ComputationalGrid.DY
+    * wbx.ComputationalGrid.NX
+    * wbx.ComputationalGrid.NY
+)
+
+prev_total_water_balance = 0.0
+
+for i in range(20):
+    if verbose:
+        print("======================================================")
+        print(f"Timestep {i}")
+        print("======================================================")
+
+    total_water_in_domain = 0.0
+
+    filename = os.path.join(new_output_dir_name, f"{run_name}.out.press.{i:05d}.pfb")
+    pressure = read_pfb(filename)
+    header = ParflowBinaryReader(filename).read_header()
+    dx, dy, dz = header["dx"], header["dy"], header["dz"]
+    surface_storage = hydrology.calculate_surface_storage(pressure, dx, dy, mask)
+    write_pfb(f"surface_storage.{i}.pfb", surface_storage)
+    total_surface_storage = np.sum(surface_storage)
+    if verbose:
+        print(f"Surface storage\t\t\t\t\t : {total_surface_storage:.16e}")
+
+    total_water_in_domain += total_surface_storage
+
+    filename = f"{run_name}.out.satur.{i:05d}.pfb"
+    saturation = read_pfb(os.path.join(new_output_dir_name, filename))
+    water_table_depth = hydrology.calculate_water_table_depth(saturation, top, dz)
+    write_pfb(f"water_table_depth.{i}.pfb", water_table_depth)
+    nz = header["nz"]
+    subsurface_storage = hydrology.calculate_subsurface_storage(
+        porosity,
+        pressure,
+        saturation,
+        specific_storage,
+        dx,
+        dy,
+        np.array([dz] * nz),
+        mask,
+    )
+    write_pfb(f"subsurface_storage.{i}.pfb", subsurface_storage)
+    total_subsurface_storage = np.sum(subsurface_storage)
+    if verbose:
+        print(f"Subsurface storage\t\t\t\t : {total_subsurface_storage:.16e}")
+
+    total_water_in_domain += total_subsurface_storage
+
+    if verbose:
+        print(f"Total water in domain\t\t\t\t : {total_water_in_domain:.16e}")
+        print("")
+
+    total_surface_runoff = 0.0
+    if i > 0:
+        surface_runoff = hydrology.calculate_overland_flow(
+            pressure,
+            slope_x[0],
+            slope_y[0],
+            mannings[0],
+            dx,
+            dy,
+            flow_method="OverlandFlow",
+            mask=mask,
+        )
+        total_surface_runoff = surface_runoff * wbx.TimingInfo.DumpInterval
+        if verbose:
+            print(f"Surface runoff from pftools\t\t\t : {total_surface_runoff:.16e}")
+
+        filename = f"{run_name}.out.overlandsum.{i:05d}.pfb"
+        surface_runoff2 = read_pfb(os.path.join(new_output_dir_name, filename))
+        total_surface_runoff2 = np.sum(surface_runoff2)
+        if verbose:
+            print(
+                f"Surface runoff from pfsimulator\t\t\t : {total_surface_runoff2:.16e}"
+            )
+
+        if not pf_test_equal(
+            total_surface_runoff, total_surface_runoff2, "Surface runoff comparison"
+        ):
+            passed = False
+
+    if i < 1:
+        bc_index = 0
+    elif 1 <= i < 7:
+        bc_index = i - 1
+    else:
+        bc_index = 6
+
+    bc_flux = bc_dict[bc_index]
+    boundary_flux = bc_flux * surface_area_of_domain * wbx.TimingInfo.DumpInterval
+
+    if verbose:
+        print(f"BC flux\t\t\t\t\t\t : {boundary_flux:.16e}")
+
+    expected_difference = boundary_flux + total_surface_runoff
+    if verbose:
+        print(f"Total Flux\t\t\t\t\t : {expected_difference:.16e}")
+
+    if i > 0:
+        if verbose:
+            print("")
+            print(
+                f"Diff from prev total\t\t\t\t : {total_water_in_domain - prev_total_water_balance:.16e}"
+            )
+
+        if expected_difference != 0.0:
+            percent_diff = (
+                abs(
+                    (prev_total_water_balance - total_water_in_domain)
+                    - expected_difference
+                )
+                / abs(expected_difference)
+                * 100
+            )
+            if verbose:
+                print(
+                    f"Percent diff from expected difference\t\t : {percent_diff:.12e}"
+                )
+
+        expected_water_balance = prev_total_water_balance - expected_difference
+        percent_diff = abs(
+            (total_water_in_domain - expected_water_balance)
+            / expected_water_balance
+            * 100
+        )
+        if verbose:
+            print(f"Percent diff from expected total water sum\t : {percent_diff:.12e}")
+
+        if percent_diff > 0.005:
+            print("Error: Water balance is not correct")
+            passed = False
+
+    prev_total_water_balance = total_water_in_domain
+
+if verbose:
+    print("\n\n")
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
diff --git a/test/python/water_balance_y.py b/test/python/water_balance_y.py
index ab99fe676..4899e20da 100644
--- a/test/python/water_balance_y.py
+++ b/test/python/water_balance_y.py
@@ -1,52 +1,61 @@
-#---------------------------------------------------------
+# ---------------------------------------------------------
 #  This runs the tilted-v catchment problem
 #  similar to that in Kollet and Maxwell (2006) AWR
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Import ParFlow
+# ---------------------------------------------------------
+
+import sys
+import os
+import numpy as np
+from parflow import Run
+from parflow.tools.fs import mkdir, get_absolute_path
+from parflow.tools.io import read_pfb, write_pfb, ParflowBinaryReader
+from parflow.tools.top import compute_top
+from parflow.tools import hydrology
+from parflow.tools.compare import pf_test_equal
+
+run_name = "water_balance"
+
+wby = Run(run_name, __file__)
+
+# ---------------------------------------------------------
 # Some controls for the test
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-#---------------------------------------------------------
-# Control slopes 
-#-1 = slope to lower-y
+# ---------------------------------------------------------
+# Control slopes
+# -1 = slope to lower-y
 # 0 = flat top (no overland flow)
-# 1 = slope to upper-y 
-#---------------------------------------------------------
+# 1 = slope to upper-y
+# ---------------------------------------------------------
 
 use_slopes = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Flux on the top surface
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 rain_flux = -0.05
 rec_flux = 0.0
 
-#---------------------------------------------------------
-# Import ParFlow
-#---------------------------------------------------------
-
-from parflow import Run
-from parflow.tools.fs import mkdir, get_absolute_path
-
-wby = Run("water_balance_y", __file__)
-
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 wby.FileVersion = 4
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Processor topology
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 wby.Process.Topology.P = 1
 wby.Process.Topology.Q = 1
 wby.Process.Topology.R = 1
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Computational Grid
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 wby.ComputationalGrid.Lower.X = 0.0
 wby.ComputationalGrid.Lower.Y = 0.0
@@ -60,25 +69,25 @@
 wby.ComputationalGrid.DY = 10.0
 wby.ComputationalGrid.DZ = 0.05
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # The Names of the GeomInputs
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-wby.GeomInput.Names = 'domaininput leftinput rightinput channelinput'
+wby.GeomInput.Names = "domaininput leftinput rightinput channelinput"
 
-wby.GeomInput.domaininput.GeomName = 'domain'
-wby.GeomInput.leftinput.GeomName = 'left'
-wby.GeomInput.rightinput.GeomName = 'right'
-wby.GeomInput.channelinput.GeomName = 'channel'
+wby.GeomInput.domaininput.GeomName = "domain"
+wby.GeomInput.leftinput.GeomName = "left"
+wby.GeomInput.rightinput.GeomName = "right"
+wby.GeomInput.channelinput.GeomName = "channel"
 
-wby.GeomInput.domaininput.InputType = 'Box'
-wby.GeomInput.leftinput.InputType = 'Box'
-wby.GeomInput.rightinput.InputType = 'Box'
-wby.GeomInput.channelinput.InputType = 'Box'
+wby.GeomInput.domaininput.InputType = "Box"
+wby.GeomInput.leftinput.InputType = "Box"
+wby.GeomInput.rightinput.InputType = "Box"
+wby.GeomInput.channelinput.InputType = "Box"
 
-#---------------------------------------------------------
-# Domain Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Domain Geometry
+# ---------------------------------------------------------
 
 wby.Geom.domain.Lower.X = 0.0
 wby.Geom.domain.Lower.Y = 0.0
@@ -87,11 +96,11 @@
 wby.Geom.domain.Upper.X = 300.0
 wby.Geom.domain.Upper.Y = 300.0
 wby.Geom.domain.Upper.Z = 1.5
-wby.Geom.domain.Patches = 'x_lower x_upper y_lower y_upper z_lower z_upper'
+wby.Geom.domain.Patches = "x_lower x_upper y_lower y_upper z_lower z_upper"
 
-#---------------------------------------------------------
-# Left Slope Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Left Slope Geometry
+# ---------------------------------------------------------
 
 wby.Geom.left.Lower.X = 0.0
 wby.Geom.left.Lower.Y = 0.0
@@ -101,9 +110,9 @@
 wby.Geom.left.Upper.Y = 300.0
 wby.Geom.left.Upper.Z = 1.5
 
-#---------------------------------------------------------
-# Right Slope Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Right Slope Geometry
+# ---------------------------------------------------------
 
 wby.Geom.right.Lower.X = 160.0
 wby.Geom.right.Lower.Y = 0.0
@@ -113,9 +122,9 @@
 wby.Geom.right.Upper.Y = 300.0
 wby.Geom.right.Upper.Z = 1.5
 
-#---------------------------------------------------------
-# Channel Geometry 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Channel Geometry
+# ---------------------------------------------------------
 
 wby.Geom.channel.Lower.X = 140.0
 wby.Geom.channel.Lower.Y = 0.0
@@ -125,11 +134,11 @@
 wby.Geom.channel.Upper.Y = 300.0
 wby.Geom.channel.Upper.Z = 1.5
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Perm
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wby.Geom.Perm.Names = 'left right channel'
+wby.Geom.Perm.Names = "left right channel"
 
 # Values in m/hour
 
@@ -137,9 +146,9 @@
 # the following is ignored if the perm.type "Constant" settings are not
 # commented out, below.
 
-wby.Geom.left.Perm.Type = 'TurnBands'
-wby.Geom.left.Perm.LambdaX = 50.
-wby.Geom.left.Perm.LambdaY = 50.
+wby.Geom.left.Perm.Type = "TurnBands"
+wby.Geom.left.Perm.LambdaX = 50.0
+wby.Geom.left.Perm.LambdaY = 50.0
 wby.Geom.left.Perm.LambdaZ = 0.5
 wby.Geom.left.Perm.GeomMean = 0.01
 
@@ -149,12 +158,12 @@
 wby.Geom.left.Perm.KMax = 100.0
 wby.Geom.left.Perm.DelK = 0.2
 wby.Geom.left.Perm.Seed = 33333
-wby.Geom.left.Perm.LogNormal = 'Log'
-wby.Geom.left.Perm.StratType = 'Bottom'
+wby.Geom.left.Perm.LogNormal = "Log"
+wby.Geom.left.Perm.StratType = "Bottom"
 
-wby.Geom.right.Perm.Type = 'TurnBands'
-wby.Geom.right.Perm.LambdaX = 50.
-wby.Geom.right.Perm.LambdaY = 50.
+wby.Geom.right.Perm.Type = "TurnBands"
+wby.Geom.right.Perm.LambdaX = 50.0
+wby.Geom.right.Perm.LambdaY = 50.0
 wby.Geom.right.Perm.LambdaZ = 0.5
 wby.Geom.right.Perm.GeomMean = 0.05
 
@@ -164,8 +173,8 @@
 wby.Geom.right.Perm.KMax = 100.0
 wby.Geom.right.Perm.DelK = 0.2
 wby.Geom.right.Perm.Seed = 13333
-wby.Geom.right.Perm.LogNormal = 'Log'
-wby.Geom.right.Perm.StratType = 'Bottom'
+wby.Geom.right.Perm.LogNormal = "Log"
+wby.Geom.right.Perm.StratType = "Bottom"
 
 # hydraulic conductivity is very low, but not zero, top node will have to saturate
 # before overland flow can begin and will be driven by hortonian flow
@@ -173,135 +182,135 @@
 # turning bands above.  Run time increases quite a bit with a heterogeneous
 # subsurface
 
-wby.Geom.left.Perm.Type = 'Constant'
+wby.Geom.left.Perm.Type = "Constant"
 wby.Geom.left.Perm.Value = 0.001
 
-wby.Geom.right.Perm.Type = 'Constant'
+wby.Geom.right.Perm.Type = "Constant"
 wby.Geom.right.Perm.Value = 0.01
 
-wby.Geom.channel.Perm.Type = 'Constant'
+wby.Geom.channel.Perm.Type = "Constant"
 wby.Geom.channel.Perm.Value = 0.00001
 
-wby.Perm.TensorType = 'TensorByGeom'
+wby.Perm.TensorType = "TensorByGeom"
 
-wby.Geom.Perm.TensorByGeom.Names = 'domain'
+wby.Geom.Perm.TensorByGeom.Names = "domain"
 
 wby.Geom.domain.Perm.TensorValX = 1.0
 wby.Geom.domain.Perm.TensorValY = 1.0
 wby.Geom.domain.Perm.TensorValZ = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Specific Storage
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wby.SpecificStorage.Type = 'Constant'
-wby.SpecificStorage.GeomNames = 'domain'
+wby.SpecificStorage.Type = "Constant"
+wby.SpecificStorage.GeomNames = "domain"
 wby.Geom.domain.SpecificStorage.Value = 1.0e-4
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phases
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wby.Phase.Names = 'water'
+wby.Phase.Names = "water"
 
-wby.Phase.water.Density.Type = 'Constant'
+wby.Phase.water.Density.Type = "Constant"
 wby.Phase.water.Density.Value = 1.0
 
-wby.Phase.water.Viscosity.Type = 'Constant'
+wby.Phase.water.Viscosity.Type = "Constant"
 wby.Phase.water.Viscosity.Value = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Contaminants
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wby.Contaminants.Names = ''
+wby.Contaminants.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Retardation
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wby.Geom.Retardation.GeomNames = ''
+wby.Geom.Retardation.GeomNames = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Gravity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 wby.Gravity = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Setup timing info
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 wby.TimingInfo.BaseUnit = 0.1
 wby.TimingInfo.StartCount = 0
 wby.TimingInfo.StartTime = 0.0
 wby.TimingInfo.StopTime = 2.0
 wby.TimingInfo.DumpInterval = 0.1
-wby.TimeStep.Type = 'Constant'
+wby.TimeStep.Type = "Constant"
 wby.TimeStep.Value = 0.1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Porosity
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wby.Geom.Porosity.GeomNames = 'left right channel'
+wby.Geom.Porosity.GeomNames = "left right channel"
 
-wby.Geom.left.Porosity.Type = 'Constant'
+wby.Geom.left.Porosity.Type = "Constant"
 wby.Geom.left.Porosity.Value = 0.25
 
-wby.Geom.right.Porosity.Type = 'Constant'
+wby.Geom.right.Porosity.Type = "Constant"
 wby.Geom.right.Porosity.Value = 0.25
 
-wby.Geom.channel.Porosity.Type = 'Constant'
+wby.Geom.channel.Porosity.Type = "Constant"
 wby.Geom.channel.Porosity.Value = 0.01
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Domain
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wby.Domain.GeomName = 'domain'
+wby.Domain.GeomName = "domain"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Relative Permeability
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wby.Phase.RelPerm.Type = 'VanGenuchten'
-wby.Phase.RelPerm.GeomNames = 'domain'
+wby.Phase.RelPerm.Type = "VanGenuchten"
+wby.Phase.RelPerm.GeomNames = "domain"
 
 wby.Geom.domain.RelPerm.Alpha = 6.0
-wby.Geom.domain.RelPerm.N = 2.
+wby.Geom.domain.RelPerm.N = 2.0
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Saturation
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-wby.Phase.Saturation.Type = 'VanGenuchten'
-wby.Phase.Saturation.GeomNames = 'domain'
+wby.Phase.Saturation.Type = "VanGenuchten"
+wby.Phase.Saturation.GeomNames = "domain"
 
 wby.Geom.domain.Saturation.Alpha = 6.0
-wby.Geom.domain.Saturation.N = 2.
+wby.Geom.domain.Saturation.N = 2.0
 wby.Geom.domain.Saturation.SRes = 0.2
 wby.Geom.domain.Saturation.SSat = 1.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Wells
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wby.Wells.Names = ''
+wby.Wells.Names = ""
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Time Cycles
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wby.Cycle.Names = 'constant rainrec'
-wby.Cycle.constant.Names = 'alltime'
+wby.Cycle.Names = "constant rainrec"
+wby.Cycle.constant.Names = "alltime"
 wby.Cycle.constant.alltime.Length = 1
 wby.Cycle.constant.Repeat = -1
 
 # rainfall and recession time periods are defined here
 # rain for 1 hour, recession for 2 hours
 
-wby.Cycle.rainrec.Names = 'r0 r1 r2 r3 r4 r5 r6'
+wby.Cycle.rainrec.Names = "r0 r1 r2 r3 r4 r5 r6"
 wby.Cycle.rainrec.r0.Length = 1
 wby.Cycle.rainrec.r1.Length = 1
 wby.Cycle.rainrec.r2.Length = 1
@@ -312,34 +321,34 @@
 
 wby.Cycle.rainrec.Repeat = 1
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Boundary Conditions: Pressure
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 wby.BCPressure.PatchNames = wby.Geom.domain.Patches
 
-wby.Patch.x_lower.BCPressure.Type = 'FluxConst'
-wby.Patch.x_lower.BCPressure.Cycle = 'constant'
+wby.Patch.x_lower.BCPressure.Type = "FluxConst"
+wby.Patch.x_lower.BCPressure.Cycle = "constant"
 wby.Patch.x_lower.BCPressure.alltime.Value = 0.0
 
-wby.Patch.y_lower.BCPressure.Type = 'FluxConst'
-wby.Patch.y_lower.BCPressure.Cycle = 'constant'
+wby.Patch.y_lower.BCPressure.Type = "FluxConst"
+wby.Patch.y_lower.BCPressure.Cycle = "constant"
 wby.Patch.y_lower.BCPressure.alltime.Value = 0.0
 
-wby.Patch.z_lower.BCPressure.Type = 'FluxConst'
-wby.Patch.z_lower.BCPressure.Cycle = 'constant'
+wby.Patch.z_lower.BCPressure.Type = "FluxConst"
+wby.Patch.z_lower.BCPressure.Cycle = "constant"
 wby.Patch.z_lower.BCPressure.alltime.Value = 0.0
 
-wby.Patch.x_upper.BCPressure.Type = 'FluxConst'
-wby.Patch.x_upper.BCPressure.Cycle = 'constant'
+wby.Patch.x_upper.BCPressure.Type = "FluxConst"
+wby.Patch.x_upper.BCPressure.Cycle = "constant"
 wby.Patch.x_upper.BCPressure.alltime.Value = 0.0
 
-wby.Patch.y_upper.BCPressure.Type = 'FluxConst'
-wby.Patch.y_upper.BCPressure.Cycle = 'constant'
+wby.Patch.y_upper.BCPressure.Type = "FluxConst"
+wby.Patch.y_upper.BCPressure.Cycle = "constant"
 wby.Patch.y_upper.BCPressure.alltime.Value = 0.0
 
-wby.Patch.z_upper.BCPressure.Type = 'OverlandFlow'
-wby.Patch.z_upper.BCPressure.Cycle = 'rainrec'
+wby.Patch.z_upper.BCPressure.Type = "OverlandFlow"
+wby.Patch.z_upper.BCPressure.Cycle = "rainrec"
 wby.Patch.z_upper.BCPressure.r0.Value = rec_flux
 wby.Patch.z_upper.BCPressure.r1.Value = rec_flux
 wby.Patch.z_upper.BCPressure.r2.Value = rain_flux
@@ -348,115 +357,281 @@
 wby.Patch.z_upper.BCPressure.r5.Value = rec_flux
 wby.Patch.z_upper.BCPressure.r6.Value = rec_flux
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in x-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-wby.TopoSlopesX.Type = 'Constant'
-wby.TopoSlopesX.GeomNames = 'left right channel'
+wby.TopoSlopesX.Type = "Constant"
+wby.TopoSlopesX.GeomNames = "left right channel"
 if use_slopes > 0:
-  wby.TopoSlopesX.Geom.left.Value = -0.005
-  wby.TopoSlopesX.Geom.right.Value = 0.005
-  wby.TopoSlopesX.Geom.channel.Value = 0.00
+    wby.TopoSlopesX.Geom.left.Value = -0.005
+    wby.TopoSlopesX.Geom.right.Value = 0.005
+    wby.TopoSlopesX.Geom.channel.Value = 0.00
 else:
-  wby.TopoSlopesX.Geom.left.Value = 0.00
-  wby.TopoSlopesX.Geom.right.Value = 0.00
-  wby.TopoSlopesX.Geom.channel.Value = 0.00
+    wby.TopoSlopesX.Geom.left.Value = 0.00
+    wby.TopoSlopesX.Geom.right.Value = 0.00
+    wby.TopoSlopesX.Geom.channel.Value = 0.00
 
-#---------------------------------------------------------
+# ---------------------------------------------------------
 # Topo slopes in y-direction
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
-wby.TopoSlopesY.Type = 'Constant'
-wby.TopoSlopesY.GeomNames = 'left right channel'
+wby.TopoSlopesY.Type = "Constant"
+wby.TopoSlopesY.GeomNames = "left right channel"
 if use_slopes > 0:
-  wby.TopoSlopesY.Geom.left.Value = 0.000
-  wby.TopoSlopesY.Geom.right.Value = 0.000
-  wby.TopoSlopesY.Geom.channel.Value = 0.001*use_slopes
+    wby.TopoSlopesY.Geom.left.Value = 0.000
+    wby.TopoSlopesY.Geom.right.Value = 0.000
+    wby.TopoSlopesY.Geom.channel.Value = 0.001 * use_slopes
 else:
-  wby.TopoSlopesY.Geom.left.Value = 0.000
-  wby.TopoSlopesY.Geom.right.Value = 0.000
-  wby.TopoSlopesY.Geom.channel.Value = 0.000
+    wby.TopoSlopesY.Geom.left.Value = 0.000
+    wby.TopoSlopesY.Geom.right.Value = 0.000
+    wby.TopoSlopesY.Geom.channel.Value = 0.000
 
-#---------------------------------------------------------
-# Mannings coefficient 
-#---------------------------------------------------------
+# ---------------------------------------------------------
+# Mannings coefficient
+# ---------------------------------------------------------
 
-wby.Mannings.Type = 'Constant'
-wby.Mannings.GeomNames = 'left right channel'
-wby.Mannings.Geom.left.Value = 5.e-6
-wby.Mannings.Geom.right.Value = 5.e-6
-wby.Mannings.Geom.channel.Value = 1.e-6
+wby.Mannings.Type = "Constant"
+wby.Mannings.GeomNames = "left right channel"
+wby.Mannings.Geom.left.Value = 5.0e-6
+wby.Mannings.Geom.right.Value = 5.0e-6
+wby.Mannings.Geom.channel.Value = 1.0e-6
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Phase sources:
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wby.PhaseSources.water.Type = 'Constant'
-wby.PhaseSources.water.GeomNames = 'domain'
+wby.PhaseSources.water.Type = "Constant"
+wby.PhaseSources.water.GeomNames = "domain"
 wby.PhaseSources.water.Geom.domain.Value = 0.0
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Exact solution specification for error calculations
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wby.KnownSolution = 'NoKnownSolution'
+wby.KnownSolution = "NoKnownSolution"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Set solver parameters
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-wby.Solver = 'Richards'
+wby.Solver = "Richards"
 wby.Solver.MaxIter = 2500
 
-wby.Solver.AbsTol = 1E-12
+wby.Solver.AbsTol = 1e-12
 wby.Solver.Nonlinear.MaxIter = 300
 wby.Solver.Nonlinear.ResidualTol = 1e-12
-wby.Solver.Nonlinear.EtaChoice = 'Walker1'
-wby.Solver.Nonlinear.EtaChoice = 'EtaConstant'
+wby.Solver.Nonlinear.EtaChoice = "Walker1"
+wby.Solver.Nonlinear.EtaChoice = "EtaConstant"
 wby.Solver.Nonlinear.EtaValue = 0.001
 wby.Solver.Nonlinear.UseJacobian = False
 wby.Solver.Nonlinear.DerivativeEpsilon = 1e-16
 wby.Solver.Nonlinear.StepTol = 1e-30
-wby.Solver.Nonlinear.Globalization = 'LineSearch'
+wby.Solver.Nonlinear.Globalization = "LineSearch"
 wby.Solver.Linear.KrylovDimension = 20
 wby.Solver.Linear.MaxRestart = 2
 
-wby.Solver.Linear.Preconditioner = 'PFMG'
+wby.Solver.Linear.Preconditioner = "PFMG"
 wby.Solver.Linear.Preconditioner.PFMG.MaxIter = 1
-wby.Solver.Linear.Preconditioner.PFMG.Smoother = 'RBGaussSeidelNonSymmetric'
+wby.Solver.Linear.Preconditioner.PFMG.Smoother = "RBGaussSeidelNonSymmetric"
 wby.Solver.Linear.Preconditioner.PFMG.NumPreRelax = 1
 wby.Solver.Linear.Preconditioner.PFMG.NumPostRelax = 1
 
-wby.Solver.WriteSiloSubsurfData = True
-wby.Solver.WriteSiloPressure = True
-wby.Solver.WriteSiloSaturation = True
-wby.Solver.WriteSiloConcentration = True
-wby.Solver.WriteSiloSlopes = True
-wby.Solver.WriteSiloMask = True
-wby.Solver.WriteSiloEvapTrans = True
-wby.Solver.WriteSiloEvapTransSum = True
-wby.Solver.WriteSiloOverlandSum = True
-wby.Solver.WriteSiloMannings = True
-wby.Solver.WriteSiloSpecificStorage = True
-
-#---------------------------------------------------------
+wby.Solver.PrintSubsurfData = True
+wby.Solver.PrintConcentration = True
+wby.Solver.PrintSlopes = True
+wby.Solver.PrintEvapTrans = True
+wby.Solver.PrintEvapTransSum = True
+wby.Solver.PrintOverlandSum = True
+wby.Solver.PrintMannings = True
+wby.Solver.PrintSpecificStorage = True
+
+# ---------------------------------------------------------
 # Initial conditions: water pressure
-#---------------------------------------------------------
+# ---------------------------------------------------------
 
 # set water table to be at the bottom of the domain, the top layer is initially dry
-wby.ICPressure.Type = 'HydroStaticPatch'
-wby.ICPressure.GeomNames = 'domain'
+wby.ICPressure.Type = "HydroStaticPatch"
+wby.ICPressure.GeomNames = "domain"
 
 wby.Geom.domain.ICPressure.Value = -3.0
 
-wby.Geom.domain.ICPressure.RefGeom = 'domain'
-wby.Geom.domain.ICPressure.RefPatch = 'z_upper'
+wby.Geom.domain.ICPressure.RefGeom = "domain"
+wby.Geom.domain.ICPressure.RefPatch = "z_upper"
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Run and Unload the ParFlow output files
-#-----------------------------------------------------------------------------
-
-dir_name = get_absolute_path('test_output/wby')
-mkdir(dir_name)
-wby.run(working_directory=dir_name)
+# -----------------------------------------------------------------------------
+
+new_output_dir_name = get_absolute_path("test_output/water_balance_x")
+correct_output_dir_name = get_absolute_path("../correct_output")
+mkdir(new_output_dir_name)
+wby.run(working_directory=new_output_dir_name)
+
+passed = True
+verbose = True
+
+bc_dict = {
+    0: wby.Patch.z_upper.BCPressure.r0.Value,
+    1: wby.Patch.z_upper.BCPressure.r1.Value,
+    2: wby.Patch.z_upper.BCPressure.r2.Value,
+    3: wby.Patch.z_upper.BCPressure.r3.Value,
+    4: wby.Patch.z_upper.BCPressure.r4.Value,
+    5: wby.Patch.z_upper.BCPressure.r5.Value,
+    6: wby.Patch.z_upper.BCPressure.r6.Value,
+}
+
+slope_x = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.slope_x.pfb"))
+slope_y = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.slope_y.pfb"))
+mannings = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.mannings.pfb"))
+specific_storage = read_pfb(
+    os.path.join(new_output_dir_name, f"{run_name}.out.specific_storage.pfb")
+)
+porosity = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.porosity.pfb"))
+mask = read_pfb(os.path.join(new_output_dir_name, f"{run_name}.out.mask.pfb"))
+top = compute_top(mask)
+
+
+surface_area_of_domain = (
+    wby.ComputationalGrid.DX
+    * wby.ComputationalGrid.DY
+    * wby.ComputationalGrid.NX
+    * wby.ComputationalGrid.NY
+)
+prev_total_water_balance = 0.0
+
+for i in range(20):
+    if verbose:
+        print("======================================================")
+        print(f"Timestep {i}")
+        print("======================================================")
+
+    total_water_in_domain = 0.0
+
+    filename = os.path.join(new_output_dir_name, f"{run_name}.out.press.{i:05d}.pfb")
+    pressure = read_pfb(filename)
+    header = ParflowBinaryReader(filename).read_header()
+    dx, dy, dz = header["dx"], header["dy"], header["dz"]
+    surface_storage = hydrology.calculate_surface_storage(pressure, dx, dy, mask)
+    write_pfb(f"surface_storage.{i}.pfb", surface_storage)
+    total_surface_storage = np.sum(surface_storage)
+    if verbose:
+        print(f"Surface storage\t\t\t\t\t : {total_surface_storage:.16e}")
+
+    total_water_in_domain += total_surface_storage
+
+    filename = f"{run_name}.out.satur.{i:05d}.pfb"
+    saturation = read_pfb(os.path.join(new_output_dir_name, filename))
+    water_table_depth = hydrology.calculate_water_table_depth(saturation, top, dz)
+    write_pfb(f"water_table_depth.{i}.pfb", water_table_depth)
+    nz = header["nz"]
+    subsurface_storage = hydrology.calculate_subsurface_storage(
+        porosity,
+        pressure,
+        saturation,
+        specific_storage,
+        dx,
+        dy,
+        np.array([dz] * nz),
+        mask,
+    )
+    write_pfb(f"subsurface_storage.{i}.pfb", subsurface_storage)
+    total_subsurface_storage = np.sum(subsurface_storage)
+    if verbose:
+        print(f"Subsurface storage\t\t\t\t : {total_subsurface_storage:.16e}")
+
+    total_water_in_domain += total_subsurface_storage
+
+    if verbose:
+        print(f"Total water in domain\t\t\t\t : {total_water_in_domain:.16e}")
+        print("")
+
+    total_surface_runoff = 0.0
+    if i > 0:
+        surface_runoff = hydrology.calculate_overland_flow(
+            pressure,
+            slope_x[0],
+            slope_y[0],
+            mannings[0],
+            dx,
+            dy,
+            flow_method="OverlandFlow",
+            mask=mask,
+        )
+        total_surface_runoff = surface_runoff * wby.TimingInfo.DumpInterval
+        if verbose:
+            print(f"Surface runoff from pftools\t\t\t : {total_surface_runoff:.16e}")
+
+        filename = f"{run_name}.out.overlandsum.{i:05d}.pfb"
+        surface_runoff2 = read_pfb(os.path.join(new_output_dir_name, filename))
+        total_surface_runoff2 = np.sum(surface_runoff2)
+        if verbose:
+            print(
+                f"Surface runoff from pfsimulator\t\t\t : {total_surface_runoff2:.16e}"
+            )
+
+        if not pf_test_equal(
+            total_surface_runoff, total_surface_runoff2, "Surface runoff comparison"
+        ):
+            passed = False
+
+    if i < 1:
+        bc_index = 0
+    elif 1 <= i < 7:
+        bc_index = i - 1
+    else:
+        bc_index = 6
+
+    bc_flux = bc_dict[bc_index]
+    boundary_flux = bc_flux * surface_area_of_domain * wby.TimingInfo.DumpInterval
+
+    if verbose:
+        print(f"BC flux\t\t\t\t\t\t : {boundary_flux:.16e}")
+
+    expected_difference = boundary_flux + total_surface_runoff
+    if verbose:
+        print(f"Total Flux\t\t\t\t\t : {expected_difference:.16e}")
+
+    if i > 0:
+        if verbose:
+            print("")
+            print(
+                f"Diff from prev total\t\t\t\t : {total_water_in_domain - prev_total_water_balance:.16e}"
+            )
+
+        if expected_difference != 0.0:
+            percent_diff = (
+                abs(
+                    (prev_total_water_balance - total_water_in_domain)
+                    - expected_difference
+                )
+                / abs(expected_difference)
+                * 100
+            )
+            if verbose:
+                print(
+                    f"Percent diff from expected difference\t\t : {percent_diff:.12e}"
+                )
+
+        expected_water_balance = prev_total_water_balance - expected_difference
+        percent_diff = abs(
+            (total_water_in_domain - expected_water_balance)
+            / expected_water_balance
+            * 100
+        )
+        if verbose:
+            print(f"Percent diff from expected total water sum\t : {percent_diff:.12e}")
+
+        if percent_diff > 0.005:
+            print("Error: Water balance is not correct")
+            passed = False
+
+    prev_total_water_balance = total_water_in_domain
+
+if verbose:
+    print("\n\n")
+
+if passed:
+    print(f"{run_name} : PASSED")
+else:
+    print(f"{run_name} : FAILED")
diff --git a/test/tcl/CMakeLists.txt b/test/tcl/CMakeLists.txt
index d5ae3db7d..7fd8b6d86 100644
--- a/test/tcl/CMakeLists.txt
+++ b/test/tcl/CMakeLists.txt
@@ -2,7 +2,6 @@
 include(ParflowTest)
 
 set(TESTS
-  Dirichlet.tcl
   default_single.tcl
   default_richards_wells.tcl
   forsyth2.tcl
@@ -14,12 +13,13 @@ set(TESTS
   small_domain.tcl
   richards_hydrostatic_equalibrium.tcl
   LW_surface_press.tcl
-  bc_pressure_file.tcl
-  bc_flux_file.tcl
 )
 
 if(${PARFLOW_HAVE_HYPRE})
   list(APPEND TESTS
+    Dirichlet.tcl
+    bc_pressure_file.tcl
+    bc_flux_file.tcl
     default_richards.tcl
     default_richards_nocluster.tcl
     default_overland.tcl
@@ -63,8 +63,9 @@ if(${PARFLOW_HAVE_NETCDF})
   if(${PARFLOW_HAVE_HYPRE})
     #This test is failing on several platforms
     list(APPEND TESTS
-    	default_richards_with_netcdf.tcl
-    	default_richards_with_netcdf_compressed.tcl)
+      default_richards_with_netcdf.tcl
+      default_richards_with_netcdf_compressed.tcl
+      default_richards_restart_netcdf.tcl)
   endif()
 endif()
 
diff --git a/test/tcl/Makefile b/test/tcl/Makefile
index 79a39bc7a..aca715eb0 100644
--- a/test/tcl/Makefile
+++ b/test/tcl/Makefile
@@ -25,6 +25,7 @@ clean: testclean
 
 testclean:
 	@rm -f *.pfb*
+	@rm -f *.nc
 	@rm -f *.silo*
 	@rm -f *.pfsb*
 	@rm -f *.sa
diff --git a/test/tcl/clm/clm-reuse/clm-reuse.tcl b/test/tcl/clm/clm-reuse/clm-reuse.tcl
old mode 100755
new mode 100644
diff --git a/test/tcl/default_richards_restart_netcdf.tcl b/test/tcl/default_richards_restart_netcdf.tcl
new file mode 100644
index 000000000..98c73b080
--- /dev/null
+++ b/test/tcl/default_richards_restart_netcdf.tcl
@@ -0,0 +1,533 @@
+#  Testing restart using PFB and NetCDF files
+#
+# Import the ParFlow TCL package
+#
+lappend auto_path $env(PARFLOW_DIR)/bin 
+package require parflow
+namespace import Parflow::*
+
+proc remove_restart_files {} {
+
+    set files [glob -nocomplain default_richards_restart.out.press.* default_richards_restart.out.satur.*]
+    foreach path $files {
+	file delete -force -- $path
+    }
+}
+
+pfset FileVersion 4
+
+pfset Process.Topology.P        [lindex $argv 0]
+pfset Process.Topology.Q        [lindex $argv 1]
+pfset Process.Topology.R        [lindex $argv 2]
+
+#---------------------------------------------------------
+# Computational Grid
+#---------------------------------------------------------
+pfset ComputationalGrid.Lower.X                -10.0
+pfset ComputationalGrid.Lower.Y                 10.0
+pfset ComputationalGrid.Lower.Z                  1.0
+
+pfset ComputationalGrid.DX	                 8.8888888888888893
+pfset ComputationalGrid.DY                      10.666666666666666
+pfset ComputationalGrid.DZ	                 1.0
+
+pfset ComputationalGrid.NX                      18
+pfset ComputationalGrid.NY                      15
+pfset ComputationalGrid.NZ                       8
+
+#---------------------------------------------------------
+# The Names of the GeomInputs
+#---------------------------------------------------------
+pfset GeomInput.Names "domain_input background_input source_region_input \
+		       concen_region_input"
+
+
+#---------------------------------------------------------
+# Domain Geometry Input
+#---------------------------------------------------------
+pfset GeomInput.domain_input.InputType            Box
+pfset GeomInput.domain_input.GeomName             domain
+
+#---------------------------------------------------------
+# Domain Geometry
+#---------------------------------------------------------
+pfset Geom.domain.Lower.X                        -10.0 
+pfset Geom.domain.Lower.Y                         10.0
+pfset Geom.domain.Lower.Z                          1.0
+
+pfset Geom.domain.Upper.X                        150.0
+pfset Geom.domain.Upper.Y                        170.0
+pfset Geom.domain.Upper.Z                          9.0
+
+pfset Geom.domain.Patches "left right front back bottom top"
+
+#---------------------------------------------------------
+# Background Geometry Input
+#---------------------------------------------------------
+pfset GeomInput.background_input.InputType         Box
+pfset GeomInput.background_input.GeomName          background
+
+#---------------------------------------------------------
+# Background Geometry
+#---------------------------------------------------------
+pfset Geom.background.Lower.X -99999999.0
+pfset Geom.background.Lower.Y -99999999.0
+pfset Geom.background.Lower.Z -99999999.0
+
+pfset Geom.background.Upper.X  99999999.0
+pfset Geom.background.Upper.Y  99999999.0
+pfset Geom.background.Upper.Z  99999999.0
+
+
+#---------------------------------------------------------
+# Source_Region Geometry Input
+#---------------------------------------------------------
+pfset GeomInput.source_region_input.InputType      Box
+pfset GeomInput.source_region_input.GeomName       source_region
+
+#---------------------------------------------------------
+# Source_Region Geometry
+#---------------------------------------------------------
+pfset Geom.source_region.Lower.X    65.56
+pfset Geom.source_region.Lower.Y    79.34
+pfset Geom.source_region.Lower.Z     4.5
+
+pfset Geom.source_region.Upper.X    74.44
+pfset Geom.source_region.Upper.Y    89.99
+pfset Geom.source_region.Upper.Z     5.5
+
+
+#---------------------------------------------------------
+# Concen_Region Geometry Input
+#---------------------------------------------------------
+pfset GeomInput.concen_region_input.InputType       Box
+pfset GeomInput.concen_region_input.GeomName        concen_region
+
+#---------------------------------------------------------
+# Concen_Region Geometry
+#---------------------------------------------------------
+pfset Geom.concen_region.Lower.X   60.0
+pfset Geom.concen_region.Lower.Y   80.0
+pfset Geom.concen_region.Lower.Z    4.0
+
+pfset Geom.concen_region.Upper.X   80.0
+pfset Geom.concen_region.Upper.Y  100.0
+pfset Geom.concen_region.Upper.Z    6.0
+
+#-----------------------------------------------------------------------------
+# Perm
+#-----------------------------------------------------------------------------
+pfset Geom.Perm.Names "background"
+
+pfset Geom.background.Perm.Type     Constant
+pfset Geom.background.Perm.Value    4.0
+
+pfset Perm.TensorType               TensorByGeom
+
+pfset Geom.Perm.TensorByGeom.Names  "background"
+
+pfset Geom.background.Perm.TensorValX  1.0
+pfset Geom.background.Perm.TensorValY  1.0
+pfset Geom.background.Perm.TensorValZ  1.0
+
+#-----------------------------------------------------------------------------
+# Specific Storage
+#-----------------------------------------------------------------------------
+
+pfset SpecificStorage.Type            Constant
+pfset SpecificStorage.GeomNames       "domain"
+pfset Geom.domain.SpecificStorage.Value 1.0e-4
+
+#-----------------------------------------------------------------------------
+# Phases
+#-----------------------------------------------------------------------------
+
+pfset Phase.Names "water"
+
+pfset Phase.water.Density.Type	Constant
+pfset Phase.water.Density.Value	1.0
+
+pfset Phase.water.Viscosity.Type	Constant
+pfset Phase.water.Viscosity.Value	1.0
+
+#-----------------------------------------------------------------------------
+# Contaminants
+#-----------------------------------------------------------------------------
+pfset Contaminants.Names			""
+
+#-----------------------------------------------------------------------------
+# Retardation
+#-----------------------------------------------------------------------------
+pfset Geom.Retardation.GeomNames           ""
+
+#-----------------------------------------------------------------------------
+# Gravity
+#-----------------------------------------------------------------------------
+
+pfset Gravity				1.0
+
+#-----------------------------------------------------------------------------
+# Setup timing info
+#-----------------------------------------------------------------------------
+
+set timestep 0.001
+
+pfset TimingInfo.BaseUnit		1.0
+pfset TimingInfo.StartCount		0
+pfset TimingInfo.StartTime		0.0
+pfset TimingInfo.StopTime               0.05
+pfset TimingInfo.DumpInterval	       -1
+pfset TimeStep.Type                     Constant
+pfset TimeStep.Value                    $timestep
+
+#-----------------------------------------------------------------------------
+# Porosity
+#-----------------------------------------------------------------------------
+
+pfset Geom.Porosity.GeomNames          background
+
+pfset Geom.background.Porosity.Type    Constant
+pfset Geom.background.Porosity.Value   1.0
+
+#-----------------------------------------------------------------------------
+# Domain
+#-----------------------------------------------------------------------------
+pfset Domain.GeomName domain
+
+#-----------------------------------------------------------------------------
+# Relative Permeability
+#-----------------------------------------------------------------------------
+
+pfset Phase.RelPerm.Type               VanGenuchten
+pfset Phase.RelPerm.GeomNames          domain
+pfset Geom.domain.RelPerm.Alpha        0.005
+pfset Geom.domain.RelPerm.N            2.0    
+
+#---------------------------------------------------------
+# Saturation
+#---------------------------------------------------------
+
+pfset Phase.Saturation.Type            VanGenuchten
+pfset Phase.Saturation.GeomNames       domain
+pfset Geom.domain.Saturation.Alpha     0.005
+pfset Geom.domain.Saturation.N         2.0
+pfset Geom.domain.Saturation.SRes      0.2
+pfset Geom.domain.Saturation.SSat      0.99
+
+#-----------------------------------------------------------------------------
+# Wells
+#-----------------------------------------------------------------------------
+pfset Wells.Names                           ""
+
+#-----------------------------------------------------------------------------
+# Time Cycles
+#-----------------------------------------------------------------------------
+pfset Cycle.Names constant
+pfset Cycle.constant.Names		"alltime"
+pfset Cycle.constant.alltime.Length	 1
+pfset Cycle.constant.Repeat		-1
+
+#-----------------------------------------------------------------------------
+# Boundary Conditions: Pressure
+#-----------------------------------------------------------------------------
+pfset BCPressure.PatchNames "left right front back bottom top"
+
+pfset Patch.left.BCPressure.Type			DirEquilRefPatch
+pfset Patch.left.BCPressure.Cycle			"constant"
+pfset Patch.left.BCPressure.RefGeom			domain
+pfset Patch.left.BCPressure.RefPatch			bottom
+pfset Patch.left.BCPressure.alltime.Value		5.0
+
+pfset Patch.right.BCPressure.Type			DirEquilRefPatch
+pfset Patch.right.BCPressure.Cycle			"constant"
+pfset Patch.right.BCPressure.RefGeom			domain
+pfset Patch.right.BCPressure.RefPatch			bottom
+pfset Patch.right.BCPressure.alltime.Value		3.0
+
+pfset Patch.front.BCPressure.Type			FluxConst
+pfset Patch.front.BCPressure.Cycle			"constant"
+pfset Patch.front.BCPressure.alltime.Value		0.0
+
+pfset Patch.back.BCPressure.Type			FluxConst
+pfset Patch.back.BCPressure.Cycle			"constant"
+pfset Patch.back.BCPressure.alltime.Value		0.0
+
+pfset Patch.bottom.BCPressure.Type			FluxConst
+pfset Patch.bottom.BCPressure.Cycle			"constant"
+pfset Patch.bottom.BCPressure.alltime.Value		0.0
+
+pfset Patch.top.BCPressure.Type			        FluxConst
+pfset Patch.top.BCPressure.Cycle			"constant"
+pfset Patch.top.BCPressure.alltime.Value		0.0
+
+#---------------------------------------------------------
+# Topo slopes in x-direction
+#---------------------------------------------------------
+
+pfset TopoSlopesX.Type "Constant"
+pfset TopoSlopesX.GeomNames ""
+
+pfset TopoSlopesX.Geom.domain.Value 0.0
+
+#---------------------------------------------------------
+# Topo slopes in y-direction
+#---------------------------------------------------------
+
+pfset TopoSlopesY.Type "Constant"
+pfset TopoSlopesY.GeomNames ""
+
+pfset TopoSlopesY.Geom.domain.Value 0.0
+
+#---------------------------------------------------------
+# Mannings coefficient 
+#---------------------------------------------------------
+
+pfset Mannings.Type "Constant"
+pfset Mannings.GeomNames ""
+pfset Mannings.Geom.domain.Value 0.
+
+#---------------------------------------------------------
+# Initial conditions: water pressure
+#---------------------------------------------------------
+
+pfset ICPressure.Type                                   HydroStaticPatch
+pfset ICPressure.GeomNames                              domain
+pfset Geom.domain.ICPressure.Value                      3.0
+pfset Geom.domain.ICPressure.RefGeom                    domain
+pfset Geom.domain.ICPressure.RefPatch                   bottom
+
+#-----------------------------------------------------------------------------
+# Phase sources:
+#-----------------------------------------------------------------------------
+
+pfset PhaseSources.water.Type                         Constant
+pfset PhaseSources.water.GeomNames                    background
+pfset PhaseSources.water.Geom.background.Value        0.0
+
+
+#-----------------------------------------------------------------------------
+# Exact solution specification for error calculations
+#-----------------------------------------------------------------------------
+
+pfset KnownSolution                                    NoKnownSolution
+
+
+#-----------------------------------------------------------------------------
+# Set solver parameters
+#-----------------------------------------------------------------------------
+pfset Solver                                             Richards
+pfset Solver.MaxIter                                     50
+
+pfset Solver.Nonlinear.MaxIter                           10
+pfset Solver.Nonlinear.ResidualTol                       1e-9
+pfset Solver.Nonlinear.EtaChoice                         EtaConstant
+pfset Solver.Nonlinear.EtaValue                          1e-5
+pfset Solver.Nonlinear.UseJacobian                       True
+pfset Solver.Nonlinear.DerivativeEpsilon                 1e-2
+
+pfset Solver.Linear.KrylovDimension                      10
+
+pfset Solver.Linear.Preconditioner                       PFMG 
+pfset Solver.Linear.Preconditioner.MGSemi.MaxIter        1
+pfset Solver.Linear.Preconditioner.MGSemi.MaxLevels      100
+
+pfset Solver.PrintVelocities True
+
+
+#-----------------------------------------------------------------------------
+# Write pressure as NetCDF for testing restart
+#-----------------------------------------------------------------------------
+pfset NetCDF.WritePressure True
+pfset NetCDF.NumStepsPerFile 25
+
+#-----------------------------------------------------------------------------
+# Run and Unload the ParFlow output files
+#-----------------------------------------------------------------------------
+pfrun default_richards
+pfundist default_richards
+
+
+#
+# Tests 
+#
+source pftest.tcl
+set passed 1
+
+if ![pftestFile default_richards.out.perm_x.pfb "Max difference in perm_x" $sig_digits] {
+    set passed 0
+}
+if ![pftestFile default_richards.out.perm_y.pfb "Max difference in perm_y" $sig_digits] {
+    set passed 0
+}
+if ![pftestFile default_richards.out.perm_z.pfb "Max difference in perm_z" $sig_digits] {
+    set passed 0
+}
+
+for {set i 1} { $i <= 50 } {incr i} {
+    set i_string [format "%05d" $i]
+    if ![pftestFile default_richards.out.press.$i_string.pfb "Max difference in Pressure for timestep $i" $sig_digits] {
+    set passed 0
+    }
+    if ![pftestFile default_richards.out.satur.$i_string.pfb "Max difference in Saturation for timestep $i" $sig_digits] {
+    set passed 0
+    }
+    # use abs value test to prevent machine precision effects
+    set abs_value 1e-12
+    if ![pftestFileWithAbs default_richards.out.velx.$i_string.pfb "Max difference in x-velocity for timestep $i" $sig_digits $abs_value] {
+    set passed 0
+    }
+    if ![pftestFileWithAbs default_richards.out.vely.$i_string.pfb "Max difference in y-velocity for timestep $i" $sig_digits $abs_value] {
+    set passed 0
+    }
+    if ![pftestFileWithAbs default_richards.out.velz.$i_string.pfb "Max difference in z-velocity for timestep $i" $sig_digits $abs_value] {
+    set passed 0
+    }
+}
+
+
+if $passed {
+    puts "default_richards : PASSED"
+} {
+    puts "default_richards : FAILED"
+}
+
+set passed 1
+remove_restart_files
+
+#=============================================================================
+# Restart run from step 10 using PFBFile
+#=============================================================================
+set istep 10
+set initial_pressure "initial_pressure.pfb"
+file copy -force default_richards.out.press.00010.pfb $initial_pressure
+pfdist $initial_pressure
+
+pfset Solver.PrintInitialConditions False
+
+pfset ICPressure.Type                 PFBFile
+pfset ICPressure.GeomNames            domain
+pfset Geom.domain.ICPressure.FileName $initial_pressure
+pfset Geom.domain.ICPressure.RefGeom  domain
+pfset Geom.domain.ICPressure.RefPatch bottom
+
+pfset TimingInfo.StartCount $istep
+pfset TimingInfo.StartTime  [expr $istep * $timestep]
+
+pfrun default_richards_restart
+pfundist default_richards_restart
+
+set correct_output_dir "../correct_output/"
+
+for {set i 11} { $i <= 50 } {incr i} {
+    set i_string [format "%05d" $i]
+    if ![pftestFile default_richards_restart.out.press.$i_string.pfb "Max difference in Pressure for timestep $i_string" $sig_digits $correct_output_dir] {
+    set passed 0
+    }
+    if ![pftestFile default_richards_restart.out.satur.$i_string.pfb "Max difference in Saturation for timestep $i_string" $sig_digits $correct_output_dir] {
+    set passed 0
+    }
+}
+
+# Remove the temporary initial pressure for the restart
+file delete -force $initial_pressure
+
+if $passed {
+    puts "default_richards_restart PFB : PASSED"
+} {
+    puts "default_richards_restart PFB: FAILED"
+}
+
+set passed 1
+remove_restart_files
+
+#=============================================================================
+# Restart run from step 10 using NetCDF file
+#=============================================================================
+set istep 10
+set initial_pressure "default_richards.out.00001.nc"
+
+pfset Solver.PrintInitialConditions False
+
+pfset ICPressure.Type                 NCFile
+pfset ICPressure.GeomNames            domain
+pfset Geom.domain.ICPressure.FileName $initial_pressure
+pfset Geom.domain.ICPressure.TimeStep 9
+pfset Geom.domain.ICPressure.RefGeom  domain
+pfset Geom.domain.ICPressure.RefPatch bottom
+
+pfset TimingInfo.StartCount $istep
+pfset TimingInfo.StartTime  [expr $istep * $timestep]
+
+pfrun default_richards_restart
+pfundist default_richards_restart
+
+set correct_output_dir "../correct_output/"
+
+for {set i 11} { $i <= 50 } {incr i} {
+    set i_string [format "%05d" $i]
+    if ![pftestFile default_richards_restart.out.press.$i_string.pfb "Max difference in Pressure for timestep $i_string" $sig_digits $correct_output_dir] {
+    set passed 0
+    }
+    if ![pftestFile default_richards_restart.out.satur.$i_string.pfb "Max difference in Saturation for timestep $i_string" $sig_digits $correct_output_dir] {
+    set passed 0
+    }
+}
+
+if $passed {
+    puts "default_richards_restart NetCDF : PASSED"
+} {
+    puts "default_richards_restart NetCDF : FAILED"
+}
+
+set passed 1
+
+#=============================================================================
+# Restart run from step last timestep in a file.  Indices in the
+# NetCDF files may be negative values to index from last timestep.
+# This is similar to Python vector indexing.  Note this is not fully
+# automated, we still have to know the initial timestep index but the
+# negative index feature makes restartng from NetCDF files a bit
+# easier.  Given an the istep still need to know which NetCDF file
+# contains that step.  In this case we wanted to restart from the
+# first NetCDF file so istep =25 since that is the number of timesteps
+# per NetCDF files.
+# =============================================================================
+
+remove_restart_files
+
+set istep 25
+set initial_pressure "default_richards.out.00001.nc"
+
+pfset Solver.PrintInitialConditions False
+
+pfset ICPressure.Type                 NCFile
+pfset ICPressure.GeomNames            domain
+pfset Geom.domain.ICPressure.FileName $initial_pressure
+pfset Geom.domain.ICPressure.TimeStep -1
+pfset Geom.domain.ICPressure.RefGeom  domain
+pfset Geom.domain.ICPressure.RefPatch bottom
+
+pfset TimingInfo.StartCount $istep
+pfset TimingInfo.StartTime  [expr $istep * $timestep]
+
+pfrun default_richards_restart
+pfundist default_richards_restart
+
+set correct_output_dir "../correct_output/"
+
+for {set i 26} { $i <= 50 } {incr i} {
+    set i_string [format "%05d" $i]
+    if ![pftestFile default_richards_restart.out.press.$i_string.pfb "Max difference in Pressure for timestep $i_string" $sig_digits $correct_output_dir] {
+    set passed 0
+    }
+    if ![pftestFile default_richards_restart.out.satur.$i_string.pfb "Max difference in Saturation for timestep $i_string" $sig_digits $correct_output_dir] {
+    set passed 0
+    }
+}
+
+if $passed {
+    puts "default_richards_restart NetCDF negative index: PASSED"
+} {
+    puts "default_richards_restart NetCDF negative index: FAILED"
+}
+
diff --git a/test/tcl/default_richards_with_netcdf.tcl b/test/tcl/default_richards_with_netcdf.tcl
index c8e6c6e4b..169ce5162 100644
--- a/test/tcl/default_richards_with_netcdf.tcl
+++ b/test/tcl/default_richards_with_netcdf.tcl
@@ -9,14 +9,6 @@ lappend auto_path $env(PARFLOW_DIR)/bin
 package require parflow
 namespace import Parflow::*
 
-# Examples of compression options for SILO
-# Note compression only works for HDF5
-#pfset SILO.Filetype "HDF5"
-#pfset SILO.CompressionOptions "METHOD=GZIP"
-#pfset SILO.CompressionOptions "METHOD=SZIP"
-#pfset SILO.CompressionOptions "METHOD=FPZIP"
-#pfset SILO.CompressionOptions "ERRMODE=FALLBACK METHOD=GZIP"
-
 pfset FileVersion 4
 
 pfset Process.Topology.P 1
diff --git a/test/tcl/timing/clm-reuse/clm-reuse.tcl b/test/tcl/timing/clm-reuse/clm-reuse.tcl
old mode 100755
new mode 100644
diff --git a/test/tcl/washita/tcl_scripts/LW_Test.tcl b/test/tcl/washita/tcl_scripts/LW_Test.tcl
index 1ad43df84..86e2e1063 100644
--- a/test/tcl/washita/tcl_scripts/LW_Test.tcl
+++ b/test/tcl/washita/tcl_scripts/LW_Test.tcl
@@ -562,9 +562,9 @@ foreach variable $ClmVariables {
 }
 
 if $passed {
-    puts "default_single : PASSED"
+    puts "LW : PASSED"
 } {
-    puts "default_single : FAILED"
+    puts "LW : FAILED"
 }
 
 
diff --git a/test/tcl/washita/tcl_scripts/LW_Test_MGSemi.tcl b/test/tcl/washita/tcl_scripts/LW_Test_MGSemi.tcl
new file mode 100644
index 000000000..4bbda22e7
--- /dev/null
+++ b/test/tcl/washita/tcl_scripts/LW_Test_MGSemi.tcl
@@ -0,0 +1,570 @@
+# SCRIPT TO RUN LITTLE WASHITA DOMAIN WITH TERRAIN-FOLLOWING GRID
+# DETAILS:
+# Arugments are 1) runname 2) year
+
+# Import the ParFlow TCL package
+lappend   auto_path $env(PARFLOW_DIR)/bin
+package   require parflow
+namespace import Parflow::*
+
+#-----------------------------------------------------------------------------
+# Make a directory for the simulation run, files will be copied to this
+# directory for running.
+#-----------------------------------------------------------------------------
+file mkdir "Outputs"
+cd "./Outputs"
+
+pfset     FileVersion    4
+
+#-----------------------------------------------------------------------------
+# Set Processor topology 
+#-----------------------------------------------------------------------------
+pfset Process.Topology.P 1
+pfset Process.Topology.Q 1
+pfset Process.Topology.R 1
+
+#-----------------------------------------------------------------------------
+# Computational Grid
+#-----------------------------------------------------------------------------
+pfset ComputationalGrid.Lower.X           0.0
+pfset ComputationalGrid.Lower.Y           0.0
+pfset ComputationalGrid.Lower.Z           0.0 
+
+pfset ComputationalGrid.DX                1000.0
+pfset ComputationalGrid.DY                1000.0
+pfset ComputationalGrid.DZ                2.0
+
+pfset ComputationalGrid.NX                41 
+pfset ComputationalGrid.NY                41 
+pfset ComputationalGrid.NZ                50  
+
+#-----------------------------------------------------------------------------
+# Names of the GeomInputs
+#-----------------------------------------------------------------------------
+pfset GeomInput.Names                     "box_input indi_input"
+
+#-----------------------------------------------------------------------------
+# Domain Geometry Input
+#-----------------------------------------------------------------------------
+pfset GeomInput.box_input.InputType      Box
+pfset GeomInput.box_input.GeomName      domain
+
+#-----------------------------------------------------------------------------
+# Domain Geometry 
+#-----------------------------------------------------------------------------
+pfset Geom.domain.Lower.X                        0.0
+pfset Geom.domain.Lower.Y                        0.0
+pfset Geom.domain.Lower.Z                        0.0
+ 
+pfset Geom.domain.Upper.X                        41000.0
+pfset Geom.domain.Upper.Y                        41000.0
+pfset Geom.domain.Upper.Z                          100.0
+pfset Geom.domain.Patches             "x-lower x-upper y-lower y-upper z-lower z-upper"
+
+#-----------------------------------------------------------------------------
+# Indicator Geometry Input
+#-----------------------------------------------------------------------------
+pfset GeomInput.indi_input.InputType      IndicatorField
+pfset GeomInput.indi_input.GeomNames      "s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 g1 g2 g3 g4 g5 g6 g7 g8"
+pfset Geom.indi_input.FileName            "IndicatorFile_Gleeson.50z.pfb"
+
+pfset GeomInput.s1.Value                1
+pfset GeomInput.s2.Value                2
+pfset GeomInput.s3.Value                3
+pfset GeomInput.s4.Value                4
+pfset GeomInput.s5.Value                5
+pfset GeomInput.s6.Value                6
+pfset GeomInput.s7.Value                7
+pfset GeomInput.s8.Value                8
+pfset GeomInput.s9.Value                9
+pfset GeomInput.s10.Value               10
+pfset GeomInput.s11.Value               11
+pfset GeomInput.s12.Value               12
+pfset GeomInput.s13.Value               13
+pfset GeomInput.g1.Value                21
+pfset GeomInput.g2.Value                22
+pfset GeomInput.g3.Value                23
+pfset GeomInput.g4.Value                24
+pfset GeomInput.g5.Value                25
+pfset GeomInput.g6.Value                26
+pfset GeomInput.g7.Value                27
+pfset GeomInput.g8.Value                28
+
+#-----------------------------------------------------------------------------
+# Permeability (values in m/hr)
+#-----------------------------------------------------------------------------
+pfset Geom.Perm.Names                     "domain s1 s2 s3 s4 s5 s6 s7 s8 s9 g2 g3 g6 g8"
+
+pfset Geom.domain.Perm.Type           Constant
+pfset Geom.domain.Perm.Value          0.2
+
+pfset Geom.s1.Perm.Type               Constant
+pfset Geom.s1.Perm.Value              0.269022595
+
+pfset Geom.s2.Perm.Type               Constant
+pfset Geom.s2.Perm.Value              0.043630356
+
+pfset Geom.s3.Perm.Type               Constant
+pfset Geom.s3.Perm.Value              0.015841225
+
+pfset Geom.s4.Perm.Type               Constant
+pfset Geom.s4.Perm.Value              0.007582087
+
+pfset Geom.s5.Perm.Type               Constant
+pfset Geom.s5.Perm.Value              0.01818816
+
+pfset Geom.s6.Perm.Type               Constant
+pfset Geom.s6.Perm.Value              0.005009435
+
+pfset Geom.s7.Perm.Type               Constant
+pfset Geom.s7.Perm.Value              0.005492736
+
+pfset Geom.s8.Perm.Type               Constant
+pfset Geom.s8.Perm.Value              0.004675077
+
+pfset Geom.s9.Perm.Type               Constant
+pfset Geom.s9.Perm.Value              0.003386794
+
+pfset Geom.g2.Perm.Type               Constant
+pfset Geom.g2.Perm.Value              0.025
+
+pfset Geom.g3.Perm.Type               Constant
+pfset Geom.g3.Perm.Value              0.059
+
+pfset Geom.g6.Perm.Type               Constant
+pfset Geom.g6.Perm.Value              0.2
+
+pfset Geom.g8.Perm.Type              Constant
+pfset Geom.g8.Perm.Value             0.68
+
+pfset Perm.TensorType                     TensorByGeom
+pfset Geom.Perm.TensorByGeom.Names        "domain"
+pfset Geom.domain.Perm.TensorValX         1.0d0
+pfset Geom.domain.Perm.TensorValY         1.0d0
+pfset Geom.domain.Perm.TensorValZ         1.0d0
+
+#-----------------------------------------------------------------------------
+# Specific Storage
+#-----------------------------------------------------------------------------
+pfset SpecificStorage.Type                Constant
+pfset SpecificStorage.GeomNames           "domain"
+pfset Geom.domain.SpecificStorage.Value   1.0e-5
+
+#-----------------------------------------------------------------------------
+# Phases
+#-----------------------------------------------------------------------------
+pfset Phase.Names                         "water"
+pfset Phase.water.Density.Type            Constant
+pfset Phase.water.Density.Value           1.0
+pfset Phase.water.Viscosity.Type          Constant
+pfset Phase.water.Viscosity.Value         1.0
+
+#-----------------------------------------------------------------------------
+# Contaminants
+#-----------------------------------------------------------------------------
+pfset Contaminants.Names                  ""
+
+#-----------------------------------------------------------------------------
+# Gravity
+#-----------------------------------------------------------------------------
+pfset Gravity                             1.0
+
+#-----------------------------------------------------------------------------
+# Timing (time units is set by units of permeability)
+#-----------------------------------------------------------------------------
+pfset TimingInfo.BaseUnit                 1.0
+pfset TimingInfo.StartCount               0.0
+pfset TimingInfo.StartTime                0.0
+pfset TimingInfo.StopTime                 12.0
+pfset TimingInfo.DumpInterval             24.0
+pfset TimeStep.Type                       Constant
+pfset TimeStep.Value                      1.0
+
+#-----------------------------------------------------------------------------
+# Porosity
+#-----------------------------------------------------------------------------
+pfset Geom.Porosity.GeomNames             "domain s1 s2 s3 s4 s5 s6 s7 s8 s9"
+
+pfset Geom.domain.Porosity.Type          Constant
+pfset Geom.domain.Porosity.Value         0.4
+
+pfset Geom.s1.Porosity.Type    Constant
+pfset Geom.s1.Porosity.Value   0.375
+
+pfset Geom.s2.Porosity.Type    Constant
+pfset Geom.s2.Porosity.Value   0.39
+
+pfset Geom.s3.Porosity.Type    Constant
+pfset Geom.s3.Porosity.Value   0.387
+
+pfset Geom.s4.Porosity.Type    Constant
+pfset Geom.s4.Porosity.Value   0.439
+
+pfset Geom.s5.Porosity.Type    Constant
+pfset Geom.s5.Porosity.Value   0.489
+
+pfset Geom.s6.Porosity.Type    Constant
+pfset Geom.s6.Porosity.Value   0.399
+
+pfset Geom.s7.Porosity.Type    Constant
+pfset Geom.s7.Porosity.Value   0.384
+
+pfset Geom.s8.Porosity.Type            Constant
+pfset Geom.s8.Porosity.Value           0.482
+
+pfset Geom.s9.Porosity.Type            Constant
+pfset Geom.s9.Porosity.Value           0.442
+
+#-----------------------------------------------------------------------------
+# Domain
+#-----------------------------------------------------------------------------
+pfset Domain.GeomName                     "domain"
+
+#----------------------------------------------------------------------------
+# Mobility
+#----------------------------------------------------------------------------
+pfset Phase.water.Mobility.Type        Constant
+pfset Phase.water.Mobility.Value       1.0
+
+#-----------------------------------------------------------------------------
+# Wells
+#-----------------------------------------------------------------------------
+pfset Wells.Names                         ""
+
+#-----------------------------------------------------------------------------
+# Time Cycles
+#-----------------------------------------------------------------------------
+pfset Cycle.Names                         "constant"
+pfset Cycle.constant.Names                "alltime"
+pfset Cycle.constant.alltime.Length        1
+pfset Cycle.constant.Repeat               -1
+
+#-----------------------------------------------------------------------------
+# Boundary Conditions
+#-----------------------------------------------------------------------------
+pfset BCPressure.PatchNames                   [pfget Geom.domain.Patches]
+
+pfset Patch.x-lower.BCPressure.Type		      FluxConst
+pfset Patch.x-lower.BCPressure.Cycle		      "constant"
+pfset Patch.x-lower.BCPressure.alltime.Value	      0.0
+
+pfset Patch.y-lower.BCPressure.Type		      FluxConst
+pfset Patch.y-lower.BCPressure.Cycle		      "constant"
+pfset Patch.y-lower.BCPressure.alltime.Value	      0.0
+
+pfset Patch.z-lower.BCPressure.Type		      FluxConst
+pfset Patch.z-lower.BCPressure.Cycle		      "constant"
+pfset Patch.z-lower.BCPressure.alltime.Value	      0.0
+
+pfset Patch.x-upper.BCPressure.Type		      FluxConst
+pfset Patch.x-upper.BCPressure.Cycle		      "constant"
+pfset Patch.x-upper.BCPressure.alltime.Value	      0.0
+
+pfset Patch.y-upper.BCPressure.Type		      FluxConst
+pfset Patch.y-upper.BCPressure.Cycle		      "constant"
+pfset Patch.y-upper.BCPressure.alltime.Value	      0.0
+
+pfset Patch.z-upper.BCPressure.Type		      OverlandFlow
+#pfset Patch.z-upper.BCPressure.Type		      SeepageFace
+pfset Patch.z-upper.BCPressure.Cycle		      "constant"
+pfset Patch.z-upper.BCPressure.alltime.Value	      0.0
+
+#-----------------------------------------------------------------------------
+# Topo slopes in x-direction
+#-----------------------------------------------------------------------------
+pfset TopoSlopesX.Type                                "PFBFile"
+pfset TopoSlopesX.GeomNames                           "domain"
+pfset TopoSlopesX.FileName                            "LW.slopex.pfb"
+
+#-----------------------------------------------------------------------------
+# Topo slopes in y-direction
+#-----------------------------------------------------------------------------
+pfset TopoSlopesY.Type                                "PFBFile"
+pfset TopoSlopesY.GeomNames                           "domain"
+pfset TopoSlopesY.FileName                            "LW.slopey.pfb"
+
+#-----------------------------------------------------------------------------
+# Mannings coefficient
+#-----------------------------------------------------------------------------
+pfset Mannings.Type                                   "Constant"
+pfset Mannings.GeomNames                              "domain"
+pfset Mannings.Geom.domain.Value                      5.52e-6
+
+#-----------------------------------------------------------------------------
+# Relative Permeability
+#-----------------------------------------------------------------------------
+pfset Phase.RelPerm.Type                  VanGenuchten
+pfset Phase.RelPerm.GeomNames             "domain s1 s2 s3 s4 s5 s6 s7 s8 s9 "
+
+pfset Geom.domain.RelPerm.Alpha           3.5
+pfset Geom.domain.RelPerm.N               2.0
+
+pfset Geom.s1.RelPerm.Alpha        3.548
+pfset Geom.s1.RelPerm.N            4.162
+
+pfset Geom.s2.RelPerm.Alpha        3.467
+pfset Geom.s2.RelPerm.N            2.738
+
+pfset Geom.s3.RelPerm.Alpha        2.692
+pfset Geom.s3.RelPerm.N            2.445
+
+pfset Geom.s4.RelPerm.Alpha        0.501
+pfset Geom.s4.RelPerm.N            2.659
+
+pfset Geom.s5.RelPerm.Alpha        0.661
+pfset Geom.s5.RelPerm.N            2.659
+
+pfset Geom.s6.RelPerm.Alpha        1.122
+pfset Geom.s6.RelPerm.N            2.479
+
+pfset Geom.s7.RelPerm.Alpha        2.089
+pfset Geom.s7.RelPerm.N            2.318
+
+pfset Geom.s8.RelPerm.Alpha        0.832
+pfset Geom.s8.RelPerm.N            2.514
+
+pfset Geom.s9.RelPerm.Alpha        1.585
+pfset Geom.s9.RelPerm.N            2.413
+
+#-----------------------------------------------------------------------------
+# Saturation
+#-----------------------------------------------------------------------------
+pfset Phase.Saturation.Type               VanGenuchten
+pfset Phase.Saturation.GeomNames          "domain s1 s2 s3 s4 s5 s6 s7 s8 s9 "
+
+pfset Geom.domain.Saturation.Alpha        3.5
+pfset Geom.domain.Saturation.N            2.
+pfset Geom.domain.Saturation.SRes         0.2
+pfset Geom.domain.Saturation.SSat         1.0
+
+pfset Geom.s1.Saturation.Alpha        3.548
+pfset Geom.s1.Saturation.N            4.162
+pfset Geom.s1.Saturation.SRes         0.000001
+pfset Geom.s1.Saturation.SSat         1.0
+
+pfset Geom.s2.Saturation.Alpha        3.467
+pfset Geom.s2.Saturation.N            2.738
+pfset Geom.s2.Saturation.SRes         0.000001
+pfset Geom.s2.Saturation.SSat         1.0
+
+pfset Geom.s3.Saturation.Alpha        2.692
+pfset Geom.s3.Saturation.N            2.445
+pfset Geom.s3.Saturation.SRes         0.000001
+pfset Geom.s3.Saturation.SSat         1.0
+
+pfset Geom.s4.Saturation.Alpha        0.501
+pfset Geom.s4.Saturation.N            2.659
+pfset Geom.s4.Saturation.SRes         0.000001
+pfset Geom.s4.Saturation.SSat         1.0
+
+pfset Geom.s5.Saturation.Alpha        0.661
+pfset Geom.s5.Saturation.N            2.659
+pfset Geom.s5.Saturation.SRes         0.000001
+pfset Geom.s5.Saturation.SSat         1.0
+
+pfset Geom.s6.Saturation.Alpha        1.122
+pfset Geom.s6.Saturation.N            2.479
+pfset Geom.s6.Saturation.SRes         0.000001
+pfset Geom.s6.Saturation.SSat         1.0
+
+pfset Geom.s7.Saturation.Alpha        2.089
+pfset Geom.s7.Saturation.N            2.318
+pfset Geom.s7.Saturation.SRes         0.000001
+pfset Geom.s7.Saturation.SSat         1.0
+
+pfset Geom.s8.Saturation.Alpha        0.832
+pfset Geom.s8.Saturation.N            2.514
+pfset Geom.s8.Saturation.SRes         0.000001
+pfset Geom.s8.Saturation.SSat         1.0
+
+pfset Geom.s9.Saturation.Alpha        1.585
+pfset Geom.s9.Saturation.N            2.413
+pfset Geom.s9.Saturation.SRes         0.000001
+pfset Geom.s9.Saturation.SSat         1.0
+
+#-----------------------------------------------------------------------------
+# Phase sources:
+#-----------------------------------------------------------------------------
+pfset PhaseSources.water.Type                         "Constant"
+pfset PhaseSources.water.GeomNames                    "domain"
+pfset PhaseSources.water.Geom.domain.Value            0.0
+
+#----------------------------------------------------------------
+# CLM Settings:
+# ------------------------------------------------------------
+pfset Solver.LSM                                      CLM
+pfset Solver.CLM.CLMFileDir                           "clm_output/"
+pfset Solver.CLM.Print1dOut                           False
+pfset Solver.BinaryOutDir                             False
+pfset Solver.CLM.CLMDumpInterval                      1
+
+pfset Solver.CLM.MetForcing                           3D
+pfset Solver.CLM.MetFileName                          "NLDAS"
+pfset Solver.CLM.MetFilePath                          "."
+pfset Solver.CLM.MetFileNT                            24
+pfset Solver.CLM.IstepStart                           1
+
+pfset Solver.CLM.EvapBeta                             Linear
+pfset Solver.CLM.VegWaterStress                       Saturation
+pfset Solver.CLM.ResSat                               0.1
+pfset Solver.CLM.WiltingPoint                         0.12
+pfset Solver.CLM.FieldCapacity                        0.98
+pfset Solver.CLM.IrrigationType                       none
+
+#---------------------------------------------------------
+# Initial conditions: water pressure
+#---------------------------------------------------------
+pfset ICPressure.Type                                 PFBFile
+pfset ICPressure.GeomNames                            domain
+pfset Geom.domain.ICPressure.RefPatch                   z-upper
+pfset Geom.domain.ICPressure.FileName                   press.init.pfb
+
+#----------------------------------------------------------------
+# Outputs
+# ------------------------------------------------------------
+#Writing output (all pfb):
+pfset Solver.PrintSubsurfData                         False
+pfset Solver.PrintPressure                            True
+pfset Solver.PrintSaturation                          True
+pfset Solver.PrintMask                                True
+
+pfset Solver.WriteCLMBinary                           False
+pfset Solver.PrintCLM                                 True
+pfset Solver.WriteSiloSpecificStorage                 False
+pfset Solver.WriteSiloMannings                        False
+pfset Solver.WriteSiloMask                            False
+pfset Solver.WriteSiloSlopes                          False
+pfset Solver.WriteSiloSubsurfData                     False
+pfset Solver.WriteSiloPressure                        False
+pfset Solver.WriteSiloSaturation                      False
+pfset Solver.WriteSiloEvapTrans                       False
+pfset Solver.WriteSiloEvapTransSum                    False
+pfset Solver.WriteSiloOverlandSum                     False
+pfset Solver.WriteSiloCLM                             False
+
+
+#-----------------------------------------------------------------------------
+# Exact solution specification for error calculations
+#-----------------------------------------------------------------------------
+pfset KnownSolution                                   NoKnownSolution
+
+#-----------------------------------------------------------------------------
+# Set solver parameters
+#-----------------------------------------------------------------------------
+# ParFlow Solution
+pfset Solver                                          Richards
+pfset Solver.TerrainFollowingGrid                     True
+pfset Solver.Nonlinear.VariableDz                     False
+
+pfset Solver.MaxIter                                  25000
+pfset Solver.Drop                                     1E-20
+pfset Solver.AbsTol                                   1E-8
+pfset Solver.MaxConvergenceFailures                   8
+pfset Solver.Nonlinear.MaxIter                        80
+pfset Solver.Nonlinear.ResidualTol                    1e-6
+
+pfset Solver.Nonlinear.UseJacobian                       True 
+pfset Solver.Nonlinear.DerivativeEpsilon                 1e-14
+pfset Solver.Nonlinear.StepTol				 1e-30
+pfset Solver.Nonlinear.Globalization                     LineSearch
+pfset Solver.Linear.KrylovDimension                      70
+pfset Solver.Linear.MaxRestarts                           2
+
+## run test with MGSemi preconditioner
+pfset Solver.Linear.Preconditioner                       MGSemi
+pfset Solver.Linear.Preconditioner.PCMatrixType       FullJacobian
+
+
+
+#-----------------------------------------------------------------------------
+# Copy files and distribute.
+#-----------------------------------------------------------------------------
+
+# ParFlow Inputs
+set path "../../parflow_input"
+foreach file "LW.slopex LW.slopey IndicatorFile_Gleeson.50z press.init" {
+    file copy -force [format "%s/%s.pfb" $path $file] .
+}
+
+#-----------------------------------------------------------------------------
+# Distribute inputs
+#-----------------------------------------------------------------------------
+pfdist -nz 1 LW.slopex.pfb
+pfdist -nz 1 LW.slopey.pfb
+
+pfdist IndicatorFile_Gleeson.50z.pfb
+pfdist press.init.pfb
+
+#CLM Inputs
+set path "../../clm_input"
+foreach file "drv_clmin drv_vegp drv_vegm.alluv" {
+    file copy -force [format "%s/%s.dat" $path $file] .
+}
+
+set path "../../NLDAS"
+foreach file "NLDAS.DSWR.000001_to_000024 NLDAS.DLWR.000001_to_000024 NLDAS.APCP.000001_to_000024 NLDAS.Temp.000001_to_000024 NLDAS.UGRD.000001_to_000024 NLDAS.VGRD.000001_to_000024 NLDAS.Press.000001_to_000024 NLDAS.SPFH.000001_to_000024" {
+    file copy -force [format "%s/%s.pfb" $path $file] .
+    pfdist -nz 24 [format "%s.pfb" $file]
+}
+
+file delete correct_output
+file link -symbolic correct_output "../correct_output"
+
+#-----------------------------------------------------------------------------
+# Run Simulation 
+#-----------------------------------------------------------------------------
+set runname "LW"
+pfrun    $runname
+
+puts "ParFlow run Complete"
+
+#-----------------------------------------------------------------------------
+# Undistribute outputs
+#-----------------------------------------------------------------------------
+pfundist $runname
+
+set StartTime [expr int([pfget TimingInfo.StartTime])]
+set StopTime [expr int([pfget TimingInfo.StopTime])]
+
+set ClmVariables [list "eflx_lh_tot" "qflx_evap_soi" "swe_out" "eflx_lwrad_out" "qflx_evap_tot" "t_grnd" "eflx_sh_tot" "qflx_evap_veg" "t_soil" "eflx_soil_grnd" "qflx_infl" "qflx_evap_grnd" "qflx_tran_veg" ]
+for {set i $StartTime} { $i <= $StopTime } {incr i} { 
+    set step [format "%05d" $i]
+    foreach variable $ClmVariables {
+        pfundist $runname.out.$variable.$step.pfb
+    }
+}
+
+#-----------------------------------------------------------------------------
+# Verify output
+#-----------------------------------------------------------------------------
+
+source ../../../pftest.tcl
+
+set sig_digits 4
+
+set passed 1
+
+set ParflowVariables [list "satur" "press"]
+set step [format "%05d" 0]
+foreach variable $ParflowVariables {
+    set file $runname.out.$variable.$step.pfb 
+    if ![pftestFile $file "Max difference in $file" $sig_digits] {
+	set passed 0
+    }
+}
+
+set step [format "%05d" 12]
+foreach variable $ClmVariables {
+    set file $runname.out.$variable.$step.pfb 
+    if ![pftestFile $file "Max difference in $file" $sig_digits] { 
+	set passed 0 
+    } 
+}
+
+if $passed {
+    puts "LW_MGSemi : PASSED"
+} {
+    puts "LW_MGSemi : FAILED"
+}
+
+